From 916d4f5b4fc6d8620e0f81b4d4c98c10d37fb58e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 24 Jun 2024 01:25:55 -0700 Subject: [PATCH] Add undefined name warning Resolves #843. --- coconut/compiler/compiler.py | 70 +++++++++++++------ coconut/constants.py | 7 +- coconut/root.py | 2 +- .../src/cocotest/agnostic/primary_1.coco | 4 +- .../tests/src/cocotest/agnostic/tutorial.coco | 6 +- .../tests/src/cocotest/target_2/py2_test.coco | 2 +- coconut/tests/src/extras.coco | 10 +++ 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 0fb12c37c..8aaf2496c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -246,12 +246,17 @@ def import_stmt(imp_from, imp, imp_as, raw=False): ) -def imported_names(imports): - """Yields all the names imported by imports = [[imp1], [imp2, as], ...].""" +def get_imported_names(imports): + """Returns all the names imported by imports = [[imp1], [imp2, as], ...] and whether there is a star import.""" + saw_names = [] + saw_star = False for imp in imports: imp_name = imp[-1].split(".", 1)[0] - if imp_name != "*": - yield imp_name + if imp_name == "*": + saw_star = True + else: + saw_names.append(imp_name) + return saw_names, saw_star def special_starred_import_handle(imp_all=False): @@ -529,7 +534,8 @@ def reset(self, keep_state=False, filename=None): # but always overwrite temp_vars_by_key since they store locs that will be invalidated self.temp_vars_by_key = {} self.parsing_context = defaultdict(list) - self.unused_imports = defaultdict(list) + self.name_info = defaultdict(lambda: {"imported": [], "referenced": [], "assigned": []}) + self.star_import = False self.kept_lines = [] self.num_lines = 0 self.disable_name_check = False @@ -942,6 +948,11 @@ def strict_err(self, *args, **kwargs): if self.strict: raise self.make_err(CoconutStyleError, *args, **kwargs) + def strict_warn(self, *args, **kwargs): + internal_assert("extra" not in kwargs, "cannot pass extra=... to strict_warn") + if self.strict: + self.syntax_warning(*args, extra="remove --strict to dismiss", **kwargs) + def syntax_warning(self, message, original, loc, **kwargs): """Show a CoconutSyntaxWarning. Usage: self.syntax_warning(message, original, loc) @@ -1319,21 +1330,30 @@ def streamline(self, grammars, inputstring=None, force=False, inner=False): elif inputstring is not None and not inner: logger.log("No streamlining done for input of length {length}.".format(length=input_len)) + def qa_error(self, msg, original, loc): + """Strict error or warn an error that should be disabled by a NOQA comment.""" + ln = self.adjust(lineno(loc, original)) + comment = self.reformat(" ".join(self.comments[ln]), ignore_errors=True) + if not self.noqa_regex.search(comment): + self.strict_err_or_warn( + msg + " (add '# NOQA' to suppress)", + original, + loc, + endpoint=False, + ) + def run_final_checks(self, original, keep_state=False): """Run post-parsing checks to raise any necessary errors/warnings.""" - # only check for unused imports if we're not keeping state accross parses + # only check for unused imports/etc. if we're not keeping state accross parses if not keep_state: - for name, locs in self.unused_imports.items(): - for loc in locs: - ln = self.adjust(lineno(loc, original)) - comment = self.reformat(" ".join(self.comments[ln]), ignore_errors=True) - if not self.noqa_regex.search(comment): - self.strict_err_or_warn( - "found unused import " + repr(self.reformat(name, ignore_errors=True)) + " (add '# NOQA' to suppress)", - original, - loc, - endpoint=False, - ) + for name, info in self.name_info.items(): + if info["imported"] and not info["referenced"]: + for loc in info["imported"]: + self.qa_error("found unused import " + repr(self.reformat(name, ignore_errors=True)), original, loc) + if not self.star_import: # only check for undefined names when there are no * imports + if name not in all_builtins and info["referenced"] and not (info["assigned"] or info["imported"]): + for loc in info["referenced"]: + self.qa_error("found undefined name " + repr(self.reformat(name, ignore_errors=True)), original, loc) def parse_line_by_line(self, init_parser, line_parser, original): """Apply init_parser then line_parser repeatedly.""" @@ -3731,13 +3751,17 @@ def import_handle(self, original, loc, tokens): else: raise CoconutInternalException("invalid import tokens", tokens) imports = list(imports) - if imp_from == "*" or imp_from is None and "*" in imports: + imported_names, star_import = get_imported_names(imports) + self.star_import = self.star_import or star_import + if star_import: + self.strict_warn("found * import; these disable Coconut's undefined name detection", original, loc) + if imp_from == "*" or (imp_from is None and star_import): if not (len(imports) == 1 and imports[0] == "*"): raise self.make_err(CoconutSyntaxError, "only [from *] import * allowed, not from * import name", original, loc) self.syntax_warning("[from *] import * is a Coconut Easter egg and should not be used in production code", original, loc) return special_starred_import_handle(imp_all=bool(imp_from)) - for imp_name in imported_names(imports): - self.unused_imports[imp_name].append(loc) + for imp_name in imported_names: + self.name_info[imp_name]["imported"].append(loc) return self.universal_import(loc, imports, imp_from=imp_from) def complex_raise_stmt_handle(self, loc, tokens): @@ -4989,8 +5013,10 @@ def name_handle(self, original, loc, tokens, assign=False, classname=False, expr ) return typevars[name] - if not assign: - self.unused_imports.pop(name, None) + if assign: + self.name_info[name]["assigned"].append(loc) + else: + self.name_info[name]["referenced"].append(loc) if ( assign diff --git a/coconut/constants.py b/coconut/constants.py index 4b10de3b2..02a912690 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -607,15 +607,20 @@ def get_path_env_var(env_var, default): "tuple", "type", "vars", "zip", + 'Ellipsis', "__import__", '__name__', '__file__', '__annotations__', '__debug__', + '__build_class__', + '__loader__', + '__package__', + '__spec__', ) python_exceptions = ( - "BaseException", "BaseExceptionGroup", "GeneratorExit", "KeyboardInterrupt", "SystemExit", "Exception", "ArithmeticError", "FloatingPointError", "OverflowError", "ZeroDivisionError", "AssertionError", "AttributeError", "BufferError", "EOFError", "ExceptionGroup", "BaseExceptionGroup", "ImportError", "ModuleNotFoundError", "LookupError", "IndexError", "KeyError", "MemoryError", "NameError", "UnboundLocalError", "OSError", "BlockingIOError", "ChildProcessError", "ConnectionError", "BrokenPipeError", "ConnectionAbortedError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError", "FileNotFoundError", "InterruptedError", "IsADirectoryError", "NotADirectoryError", "PermissionError", "ProcessLookupError", "TimeoutError", "ReferenceError", "RuntimeError", "NotImplementedError", "RecursionError", "StopAsyncIteration", "StopIteration", "SyntaxError", "IndentationError", "TabError", "SystemError", "TypeError", "ValueError", "UnicodeError", "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeTranslateError", "Warning", "BytesWarning", "DeprecationWarning", "EncodingWarning", "FutureWarning", "ImportWarning", "PendingDeprecationWarning", "ResourceWarning", "RuntimeWarning", "SyntaxWarning", "UnicodeWarning", "UserWarning", + 'ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'BaseExceptionGroup', 'BlockingIOError', 'BrokenPipeError', 'BufferError', 'BytesWarning', 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning', 'EOFError', 'EncodingWarning', 'EnvironmentError', 'Exception', 'ExceptionGroup', 'FileExistsError', 'FileNotFoundError', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'InterruptedError', 'IsADirectoryError', 'KeyError', 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'ModuleNotFoundError', 'NameError', 'NotADirectoryError', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError', 'RecursionError', 'ReferenceError', 'ResourceWarning', 'RuntimeError', 'RuntimeWarning', 'StopAsyncIteration', 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TimeoutError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'WindowsError', 'ZeroDivisionError' ) always_keep_parse_name_prefix = "HAS_" diff --git a/coconut/root.py b/coconut/root.py index de086cbd1..0dfbd741d 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.1.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 1 +DEVELOP = 2 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary_1.coco b/coconut/tests/src/cocotest/agnostic/primary_1.coco index bfe7888cf..299e3e7e3 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_1.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_1.coco @@ -90,7 +90,7 @@ def primary_test_1() -> bool: \\assert data == 3 \\def backslash_test(): return (x) -> x - assert \(1) == 1 == backslash_test()(1) + assert \(1) == 1 == backslash_test()(1) # NOQA assert True is (\( "hello" ) == "hello" == \( @@ -100,7 +100,7 @@ def primary_test_1() -> bool: x, y): return x + y - assert multiline_backslash_test(1, 2) == 3 + assert multiline_backslash_test(1, 2) == 3 # noqa \\ assert True class one_line_class: pass assert isinstance(one_line_class(), one_line_class) diff --git a/coconut/tests/src/cocotest/agnostic/tutorial.coco b/coconut/tests/src/cocotest/agnostic/tutorial.coco index 3eeabae34..cd96b0a08 100644 --- a/coconut/tests/src/cocotest/agnostic/tutorial.coco +++ b/coconut/tests/src/cocotest/agnostic/tutorial.coco @@ -22,13 +22,15 @@ assert range(1, 5) |> product == 24 first_five_words = .split() ..> .$[:5] ..> " ".join assert first_five_words("ab cd ef gh ij kl") == "ab cd ef gh ij" -@recursive_iterator +# TODO: recursive_iterator -> recursive_generator +@recursive_iterator # noqa def fib() = (1, 1) :: map((+), fib(), fib()$[1:]) assert fib()$[:5] |> list == [1, 1, 2, 3, 5] +# TODO: parallel_map -> process_map # can't use parallel_map here otherwise each process would have to rerun all # the tutorial tests since we don't guard them behind __name__ == "__main__" -assert range(100) |> concurrent_map$(.**2) |> list |> .$[-1] == 9801 +assert range(100) |> thread_map$(.**2) |> list |> .$[-1] == 9801 def factorial(n, acc=1): match n: diff --git a/coconut/tests/src/cocotest/target_2/py2_test.coco b/coconut/tests/src/cocotest/target_2/py2_test.coco index cf8ef713e..b9711f614 100644 --- a/coconut/tests/src/cocotest/target_2/py2_test.coco +++ b/coconut/tests/src/cocotest/target_2/py2_test.coco @@ -4,5 +4,5 @@ def py2_test() -> bool: assert py_map((+)$(2), range(5)) == [2, 3, 4, 5, 6] assert py_range(5) == [0, 1, 2, 3, 4] assert not isinstance(long(1), py_int) # type: ignore - assert py_str(3) == b"3" == unicode(b"3") # type: ignore + assert py_str(3) == b"3" == unicode(b"3") # noqa # type: ignore return True diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 13c69496f..12a1560da 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -414,6 +414,16 @@ import abc except CoconutStyleError as err: assert str(err) == """found unused import 'abc' (add '# NOQA' to suppress) (remove --strict to downgrade to a warning) (line 1) import abc""" + try: + parse(""" +1 +2 + x +3 + """.strip()) + except CoconutStyleError as err: + assert str(err) == """found undefined name 'x' (add '# NOQA' to suppress) (remove --strict to downgrade to a warning) (line 2) + 2 + x + ^""" assert_raises(-> parse(""" class A(object): 1