Fix line splitting

Resolves   #818.

coconut/compiler/compiler.py

@@ -97,7 +97,7 @@
     pickleable_obj,
     checksum,
     clip,
-    logical_lines,
+    literal_lines,
     clean,
     get_target_info,
     get_clock_time,
@@ -1240,7 +1240,7 @@ def make_err(self, errtype, message, original, loc=0, ln=None, extra=None, refor
                 ln = self.outer_ln
 
         # get line indices for the error locs
-        original_lines = tuple(logical_lines(original, True))
+        original_lines = tuple(literal_lines(original, True))
         loc_line_ind = clip(lineno(loc, original) - 1, max=len(original_lines) - 1)
 
         # build the source snippet that the error is referring to
@@ -1449,7 +1449,7 @@ def prepare(self, inputstring, strip=False, nl_at_eof_check=False, **kwargs):
         if self.strict and nl_at_eof_check and inputstring and not inputstring.endswith("\n"):
             end_index = len(inputstring) - 1 if inputstring else 0
             raise self.make_err(CoconutStyleError, "missing new line at end of file", inputstring, end_index)
-        kept_lines = inputstring.splitlines()
+        kept_lines = tuple(literal_lines(inputstring))
         self.num_lines = len(kept_lines)
         if self.keep_lines:
             self.kept_lines = kept_lines
@@ -1719,7 +1719,7 @@ def operator_proc(self, inputstring, keep_state=False, **kwargs):
         """Process custom operator definitions."""
         out = []
         skips = self.copy_skips()
-        for i, raw_line in enumerate(logical_lines(inputstring, keep_newlines=True)):
+        for i, raw_line in enumerate(literal_lines(inputstring, keep_newlines=True)):
             ln = i + 1
             base_line = rem_comment(raw_line)
             stripped_line = base_line.lstrip()
@@ -1806,7 +1806,7 @@ def leading_whitespace(self, inputstring):
 
     def ind_proc(self, inputstring, **kwargs):
         """Process indentation and ensure balanced parentheses."""
-        lines = tuple(logical_lines(inputstring))
+        lines = tuple(literal_lines(inputstring))
         new = []  # new lines
         current = None  # indentation level of previous line
         levels = []  # indentation levels of all previous blocks, newest at end
@@ -1899,11 +1899,8 @@ def reind_proc(self, inputstring, ignore_errors=False, **kwargs):
         out_lines = []
         level = 0
 
-        next_line_is_fake = False
-        for line in inputstring.splitlines(True):
-            is_fake = next_line_is_fake
-            next_line_is_fake = line.endswith("\f") and line.rstrip("\f") == line.rstrip()
-
+        is_fake = False
+        for next_line_is_real, line in literal_lines(inputstring, True, yield_next_line_is_real=True):
             line, comment = split_comment(line.strip())
 
             indent, line = split_leading_indent(line)
@@ -1932,6 +1929,8 @@ def reind_proc(self, inputstring, ignore_errors=False, **kwargs):
             line = (line + comment).rstrip()
             out_lines.append(line)
 
+            is_fake = not next_line_is_real
+
         if not ignore_errors and level != 0:
             logger.log_lambda(lambda: "failed to reindent:\n" + inputstring)
             complain("non-zero final indentation level: " + repr(level))
@@ -1978,7 +1977,7 @@ def endline_repl(self, inputstring, reformatting=False, ignore_errors=False, **k
         """Add end of line comments."""
         out_lines = []
         ln = 1  # line number in pre-processed original
-        for line in logical_lines(inputstring):
+        for line in literal_lines(inputstring):
             add_one_to_ln = False
             try:
 
@@ -2331,7 +2330,7 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i
     def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, is_stmt_lambda):
         """Determines if TCO or TRE can be done and if so does it,
         handles dotted function names, and universalizes async functions."""
-        raw_lines = list(logical_lines(funcdef, True))
+        raw_lines = list(literal_lines(funcdef, True))
         def_stmt = raw_lines.pop(0)
         out = []
 
@@ -2684,7 +2683,7 @@ def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names=
         self.compile_add_code_before_regexes()
 
         out = []
-        for raw_line in inputstring.splitlines(True):
+        for raw_line in literal_lines(inputstring, True):
             bef_ind, line, aft_ind = split_leading_trailing_indent(raw_line)
 
             # look for deferred errors
@@ -2707,7 +2706,7 @@ def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names=
                 # handle any non-function code that was added before the funcdef
                 pre_def_lines = []
                 post_def_lines = []
-                funcdef_lines = list(logical_lines(funcdef, True))
+                funcdef_lines = list(literal_lines(funcdef, True))
                 for i, line in enumerate(funcdef_lines):
                     if self.def_regex.match(line):
                         pre_def_lines = funcdef_lines[:i]
@@ -3128,7 +3127,7 @@ def yield_from_handle(self, loc, tokens):
     def endline_handle(self, original, loc, tokens):
         """Add line number information to end of line."""
         endline, = tokens
-        lines = endline.splitlines(True)
+        lines = tuple(literal_lines(endline, True))
         if self.minify:
             lines = lines[0]
         out = []

coconut/compiler/util.py

@@ -88,6 +88,7 @@
     univ_open,
     ensure_dir,
     get_clock_time,
+    literal_lines,
 )
 from coconut.terminal import (
     logger,
@@ -1839,7 +1840,7 @@ def is_blank(line):
 def final_indentation_level(code):
     """Determine the final indentation level of the given code."""
     level = 0
-    for line in code.splitlines():
+    for line in literal_lines(code):
         leading_indent, _, trailing_indent = split_leading_trailing_indent(line)
         level += ind_change(leading_indent) + ind_change(trailing_indent)
     return level

coconut/exceptions.py

@@ -35,7 +35,7 @@
 from coconut.util import (
     pickleable_obj,
     clip,
-    logical_lines,
+    literal_lines,
     clean,
     get_displayable_target,
     normalize_newlines,
@@ -140,7 +140,7 @@ def message(self, message, source, point, ln, extra=None, endpoint=None, filenam
                 point_ind = getcol(point, source) - 1
                 endpoint_ind = getcol(endpoint, source) - 1
 
-                source_lines = tuple(logical_lines(source, keep_newlines=True))
+                source_lines = tuple(literal_lines(source, keep_newlines=True))
 
                 # walk the endpoint line back until it points to real text
                 while endpoint_ln > point_ln and not "".join(source_lines[endpoint_ln - 1:endpoint_ln]).strip():

coconut/root.py

@@ -26,7 +26,7 @@
 VERSION = "3.0.4"
 VERSION_NAME = None
 # False for release, int >= 1 for develop
-DEVELOP = 13
+DEVELOP = 14
 ALPHA = False  # for pre releases rather than post releases
 
 assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"

coconut/tests/src/extras.coco

@@ -116,6 +116,7 @@ def test_setup_none() -> bool:
     assert "==" not in parse("None = None")
     assert parse("(1\f+\f2)", "lenient") == "(1 + 2)" == parse("(1\f+\f2)", "eval")
     assert "Ellipsis" not in parse("x: ... = 1")
+    assert parse("linebreaks = '\x0b\x0c\x1c\x1d\x1e'")
 
     # things that don't parse correctly without the computation graph
     if USE_COMPUTATION_GRAPH:

coconut/util.py

@@ -151,8 +151,8 @@ def clip(num, min=None, max=None):
     )
 
 
-def logical_lines(text, keep_newlines=False):
-    """Iterate over the logical code lines in text."""
+def literal_lines(text, keep_newlines=False, yield_next_line_is_real=False):
+    """Iterate over the literal code lines in text."""
     prev_content = None
     for line in text.splitlines(True):
         real_line = True
@@ -163,11 +163,14 @@ def logical_lines(text, keep_newlines=False):
             if not keep_newlines:
                 line = line[:-1]
         else:
-            if prev_content is None:
-                prev_content = ""
-            prev_content += line
+            if not yield_next_line_is_real:
+                if prev_content is None:
+                    prev_content = ""
+                prev_content += line
             real_line = False
-        if real_line:
+        if yield_next_line_is_real:
+            yield real_line, line
+        elif real_line:
             if prev_content is not None:
                 line = prev_content + line
                 prev_content = None