add encoding='utf-8' to open

windows may not assume utf-8 when reading/writing to a file

Author: Anthony Yang

compiler_integration_test.py

@@ -267,7 +267,7 @@ def main():
         final_results = evaluate_and_compare_test_cases(TEST_CASES, n=1)
     print()
 
-    with open('compiler_integration_results.md', 'w') as f:
+    with open('compiler_integration_results.md', 'w', encoding='utf-8') as f:
         dump_results(final_results, f)
 
 

harmony_model_checker/dfacmp.py

@@ -3,7 +3,7 @@
 from automata.fa.dfa import DFA
 
 def parse(file):
-    with open(file) as f:
+    with open(file, encoding='utf-8') as f:
         js = json.load(f)
 
         states = { "__error__": {} }

harmony_model_checker/harmony/behavior.py

@@ -28,7 +28,7 @@ def find_error_states(transitions, final_states):
     return error_states
 
 def read_hfa(file, dfa, nfa):
-    with open(file) as fd:
+    with open(file, encoding='utf-8') as fd:
         js = json.load(fd)
         initial = js["initial"]
         states = { "{}" }
@@ -249,7 +249,7 @@ def add_edge(src, val, dst):
     dfa_error_states = find_error_states(dfa_transitions, dfa_final_states)
 
     if outputfiles["hfa"] != None:
-        with open(outputfiles["hfa"], "w") as fd:
+        with open(outputfiles["hfa"], "w", encoding='utf-8') as fd:
             names = {}
             for (idx, s) in enumerate(dfa_states):
                 names[s] = idx
@@ -295,7 +295,7 @@ def add_edge(src, val, dst):
             print("}", file=fd)
 
     if outputfiles["gv"] != None:
-        with open(outputfiles["gv"], "w") as fd:
+        with open(outputfiles["gv"], "w", encoding='utf-8') as fd:
             names = {}
             for (idx, s) in enumerate(dfa_states):
                 names[s] = idx

harmony_model_checker/harmony/brief.py

@@ -125,7 +125,7 @@ def print_macrostep(self, mas):
             self.interrupted = "interrupt" in self.lastmis and self.lastmis["interrupt"] == "True"
 
     def run(self, outputfiles, behavior):
-        with open(outputfiles["hco"]) as f:
+        with open(outputfiles["hco"], encoding='utf-8') as f:
             print("Phase 5: loading", outputfiles["hco"])
             top = json.load(f)
             assert isinstance(top, dict)

harmony_model_checker/harmony/genhtml.py

@@ -20,7 +20,7 @@ def __init__(self):
         self.js = (self_dir / "charm.js").read_text()
 
     def file_include(self, name, f):
-        with open(name) as g:
+        with open(name, encoding='utf-8') as g:
             print(g.read(), file=f)
 
     def html_megastep(self, step, tid, name, nmicrosteps, width, f):
@@ -335,7 +335,7 @@ def vars_add(self, vardir, shared):
     def run(self, outputfiles):
         # First figure out how many megasteps there are and how many threads
         lasttid = -1
-        with open(outputfiles["hco"]) as f:
+        with open(outputfiles["hco"], encoding='utf-8') as f:
             self.top = json.load(f)
             assert isinstance(self.top, dict)
             if "macrosteps" in self.top:
@@ -356,5 +356,5 @@ def run(self, outputfiles):
                         if tid >= self.nthreads:
                             self.nthreads = tid + 1
 
-        with open(outputfiles["htm"], "w") as out:
+        with open(outputfiles["htm"], "w", encoding='utf-8') as out:
             self.html(out, outputfiles)

harmony_model_checker/harmony/harmony.py

@@ -103,7 +103,7 @@ def doImport(scope, code, module):
         for dir in [ os.path.dirname(namestack[-1]), install_path + "/modules", "." ]:
             filename = dir + "/" + modname + ".hny"
             if os.path.exists(filename):
-                with open(filename) as f:
+                with open(filename, encoding='utf-8') as f:
                     load(f, filename, scope2, code)
                 found = True
                 break
@@ -2523,7 +2523,7 @@ def doImport(self, scope, code, module):
             for dir in [ os.path.dirname(namestack[-1]), install_path + "/modules", "." ]:
                 filename = dir + "/" + modname + ".hny"
                 if os.path.exists(filename):
-                    with open(filename) as f:
+                    with open(filename, encoding='utf-8') as f:
                         load(f, filename, scope2, code)
                     found = True
                     break
@@ -5380,7 +5380,7 @@ def doCompile(filenames, consts, mods, interface):
     code.append(FrameOp(("__init__", None, None, None), []))
     for fname in filenames:
         try:
-            with open(fname) as fd:
+            with open(fname, encoding='utf-8') as fd:
                 load(fd, fname, scope, code)
         except IOError:
             print("harmony: can't open", fname, file=sys.stderr)
@@ -5996,7 +5996,7 @@ def htmlcode(code, scope, f):
     print("</div>", file=f)
 
 def htmldump(nodes, code, scope, node, fulldump, verbose):
-    with open("harmony.html", "w") as f:
+    with open("harmony.html", "w", encoding='utf-8') as f:
         print("""
 <html>
   <head>
@@ -7576,7 +7576,7 @@ def main():
         code, scope = doCompile(args, consts, mods, interface)
     except HarmonyCompilerError as e:
         if parse_code_only:
-            with open(outputfiles["hvm"], "w") as f:
+            with open(outputfiles["hvm"], "w", encoding='utf-8') as f:
                 data = {
                     "errors": [e.token._as_dict()],
                     "status": "error"
@@ -7586,12 +7586,12 @@ def main():
         sys.exit(1)
 
     if parse_code_only:
-        with open(outputfiles["hvm"], "w") as f:
+        with open(outputfiles["hvm"], "w", encoding='utf-8') as f:
             f.write(json.dumps({"status": "ok"}))
         return
 
     if outputfiles["tla"] != None:
-        with open(outputfiles["tla"], "w") as fd:
+        with open(outputfiles["tla"], "w", encoding='utf-8') as fd:
             tla_translate(fd, code, scope)
 
     install_path = os.path.dirname(os.path.realpath(__file__))
@@ -7600,7 +7600,7 @@ def main():
         # see if there is a configuration file
         infile = "%s/charm.c"%install_path
         outfile = "%s/charm.exe"%install_path
-        with open(outputfiles["hvm"], "w") as fd:
+        with open(outputfiles["hvm"], "w", encoding='utf-8') as fd:
             dumpCode("json", code, scope, f=fd)
         print("Phase 2: run the model checker")
         sys.stdout.flush()

harmony_model_checker/iface.py

@@ -272,7 +272,7 @@ def main():
         usage()
 
     file = args[0]
-    with open(file) as f:
+    with open(file, encoding='utf-8') as f:
         js = json.load(f)
         parse(js, outfmt, minify);
 

harmony_model_checker/main.py

@@ -77,7 +77,7 @@ def load_file(filename: str, scope: Scope, code: Code):
     if filename in legacy_harmony.files:
         return
     legacy_harmony.namestack.append(filename)
-    with open(filename, "r") as f:
+    with open(filename, "r", encoding='utf-8') as f:
         legacy_harmony.files[filename] = f.read().split("\n")
 
     ast = parse(filename)
@@ -363,7 +363,7 @@ def main():
 
         if parse_code_only:
             data = dict(errors=[e._asdict() for e in errors], status="error")
-            with open(output_files["hvm"], "w") as fp:
+            with open(output_files["hvm"], "w", encoding='utf-8') as fp:
                 json.dump(data, fp)
         else:
             for e in errors:
@@ -372,18 +372,18 @@ def main():
         return 1
 
     if parse_code_only:
-        with open(output_files["hvm"], "w") as f:
+        with open(output_files["hvm"], "w", encoding='utf-8') as f:
             f.write(json.dumps({"status": "ok"}))
         return
 
     if output_files["tla"] is not None:
-        with open(output_files["tla"], "w") as f:
+        with open(output_files["tla"], "w", encoding='utf-8') as f:
             legacy_harmony.tla_translate(f, code, scope)
 
     # Analyze liveness of variables
     if charm_flag:
         # see if there is a configuration file
-        with open(output_files["hvm"], "w") as fd:
+        with open(output_files["hvm"], "w", encoding='utf-8') as fd:
             legacy_harmony.dumpCode("json", code, scope, f=fd)
 
         if parse_code_only:

setup.py

@@ -15,7 +15,7 @@ def read(rel_path: str) -> str:
     here = os.path.abspath(os.path.dirname(__file__))
     # intentionally *not* adding an encoding option to open, See:
     #   https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
-    with open(os.path.join(here, rel_path)) as fp:
+    with open(os.path.join(here, rel_path), encoding='utf-8') as fp:
         return fp.read()
 
 def get_version(rel_path: str) -> str: