Add explicit encoding to read_text. (#577)

* Add explicit encoding to read_text.

Closes #561

* Add explicit encoding in towncrier.create.

* Add note about UTF-8 encoding to tutorial.

* Expand the note to mention the fragments, the news file, the config file, and templates.

* Use 'utf-8' throughout for consistency.

---------

Co-authored-by: Adi Roiban <[email protected]>

docs/tutorial.rst

@@ -146,6 +146,8 @@ You should get an output similar to this::
 
 Note: if you configure a Markdown file (for example, ``filename = "CHANGES.md"``) in your configuration file, the titles will be output in Markdown format instead.
 
+Note: all files (news fragments, the news file, the configuration file, and templates) are encoded and are expected to be encoded as UTF-8.
+
 
 Producing News Files In Production
 ----------------------------------

src/towncrier/_builder.py

@@ -112,7 +112,7 @@ def find_fragments(
             full_filename = os.path.join(section_dir, basename)
             fragment_filenames.append(full_filename)
             with open(full_filename, "rb") as f:
-                data = f.read().decode("utf8", "replace")
+                data = f.read().decode("utf-8", "replace")
 
             if (ticket, category, counter) in file_content:
                 raise ValueError(

src/towncrier/_writer.py

@@ -40,7 +40,7 @@ def append_to_newsfile(
     # Leave newlines alone. This probably leads to inconsistent newlines,
     # because we've loaded existing content with universal newlines, but that's
     # the original behavior.
-    with news_file.open("w", encoding="utf8", newline="") as f:
+    with news_file.open("w", encoding="utf-8", newline="") as f:
         if header:
             f.write(header)
         # If there is no previous body that means we're writing a brand new news file.
@@ -66,7 +66,7 @@ def _figure_out_existing_content(
 
     # If we didn't use universal newlines here, we wouldn't find *start_string*
     # which usually contains a `\n`.
-    with news_file.open(encoding="utf8") as f:
+    with news_file.open(encoding="utf-8") as f:
         content = f.read()
 
     t = content.split(start_string, 1)

src/towncrier/build.py

@@ -166,7 +166,9 @@ def __main(
     click.echo("Loading template...", err=to_err)
     if isinstance(config.template, tuple):
         template = (
-            resources.files(config.template[0]).joinpath(config.template[1]).read_text()
+            resources.files(config.template[0])
+            .joinpath(config.template[1])
+            .read_text(encoding="utf-8")
         )
     else:
         with open(config.template, encoding="utf-8") as tmpl:

src/towncrier/create.py

@@ -170,7 +170,7 @@ def __main(
             click.echo("Aborted creating news fragment due to empty message.")
             ctx.exit(1)
 
-    with open(segment_file, "w") as f:
+    with open(segment_file, "w", encoding="utf-8") as f:
         f.write(content)
         if config.create_eof_newline and content and not content.endswith("\n"):
             f.write("\n")

src/towncrier/newsfragments/561.bugfix.rst

@@ -0,0 +1 @@
+Add explicit encoding to read_text.

src/towncrier/test/helpers.py

@@ -36,7 +36,7 @@ def read_pkg_resource(path: str) -> str:
     """
     Read *path* from the towncrier package.
     """
-    return (resources.files("towncrier") / path).read_text("utf8")
+    return (resources.files("towncrier") / path).read_text("utf-8")
 
 
 def with_isolated_runner(fn: Callable[..., Any]) -> Callable[..., Any]: