diff --git a/.gitignore b/.gitignore index 62dae4d5..750ce00d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ book/pp.tex # Auto-generated by Makefile's [gen] target harmony_model_checker/__init__.py + +.coverage diff --git a/Makefile b/Makefile index 1674ae18..3fbbb04f 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,25 @@ upload-test: dist upload: dist twine upload dist/* +test-unit: + coverage run -m unittest discover tests/harmony + +test-e2e: + coverage run -m unittest discover tests/e2e + +test: test-unit test-e2e + clean: + # Harmony outputs in `code` directory rm -f code/*.htm code/*.hvm code/*.hco code/*.png code/*.hfa code/*.tla code/*.gv *.html + + # Harmony outputs in `modules` directory (cd harmony_model_checker/modules; rm -f *.htm *.hvm *.hco *.png *.hfa *.tla *.gv *.html) + rm -rf compiler_integration_results.md compiler_integration_results/ + + # Package publication related outputs rm -rf build/ dist/ harmony_model_checker.egg-info/ + + # Test coverage related outputs + rm -rf .coverage htmlcov diff --git a/code/BBsema.hny b/code/BBsema.hny index 4490c5f2..19485830 100644 --- a/code/BBsema.hny +++ b/code/BBsema.hny @@ -1,7 +1,15 @@ -import synch; +from synch import Semaphore, P, V; const NSLOTS = 2; # size of bounded buffer +buf = { x:() for x in {1..NSLOTS} }; +b_in = 1; +b_out = 1; +l_in = Semaphore(1); +l_out = Semaphore(1); +n_full = Semaphore(0); +n_empty = Semaphore(NSLOTS); + def produce(item): P(?n_empty); P(?l_in); @@ -18,10 +26,3 @@ def consume(): V(?l_out); V(?n_empty); ; -buf = { x:() for x in {1..NSLOTS} }; -b_in = 1; -b_out = 1; -l_in = Semaphore(1); -l_out = Semaphore(1); -n_full = Semaphore(0); -n_empty = Semaphore(NSLOTS); diff --git a/code/BBsematest.hny b/code/BBsematest.hny index 24c02ade..46159eb3 100644 --- a/code/BBsematest.hny +++ b/code/BBsematest.hny @@ -1,4 +1,4 @@ -import BBsema; +from BBsema import produce, consume; const NPRODS = 3; # number of producers const NCONSS = 3; # number of consumers diff --git a/harmony_model_checker/harmony/ast.py b/harmony_model_checker/harmony/ast.py index 9110c2f7..068d215e 100644 --- a/harmony_model_checker/harmony/ast.py +++ b/harmony_model_checker/harmony/ast.py @@ -1270,9 +1270,9 @@ def compile(self, scope, code, stmt): for var_or_cond in self.vars_and_conds: if var_or_cond[0] == 'var': (_, var, expr, tkn, endtkn, op) = var_or_cond - stmt = self.range(token, endtkn) + stmt = self.range(tkn, endtkn) expr.compile(ns, code, stmt) - code.append(StoreVarOp(var), token, op, stmt=stmt) + code.append(StoreVarOp(var), tkn, op, stmt=stmt) self.define(ns, var) elif var_or_cond[0] == 'cond': (_, cond, token, endtkn) = var_or_cond diff --git a/requirements.txt b/requirements.txt index 5af3be68..8b4467a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ antlr-denter==1.3.1 antlr4-python3-runtime==4.9.3 automata-lib==5.0.0 +coverage==6.3.2 pydot==1.4.2 requests==2.27.1 twine==3.7.1 diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..7ef81d20 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# Harmony Tests + +Types of tests: + +- Compilation/Performance Tests +- Unit Tests + +## Compilation/Performance Tests + +## Unit Tests + diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/constants/base.hny b/tests/e2e/constants/base.hny new file mode 100644 index 00000000..5a03ff14 --- /dev/null +++ b/tests/e2e/constants/base.hny @@ -0,0 +1,3 @@ + +const C = 23 +print C diff --git a/tests/e2e/errors/bad_elif_stmt.hny b/tests/e2e/errors/bad_elif_stmt.hny new file mode 100644 index 00000000..5cd1d427 --- /dev/null +++ b/tests/e2e/errors/bad_elif_stmt.hny @@ -0,0 +1,5 @@ + +if True: + pass +else if: + pass diff --git a/tests/e2e/errors/def_label.hny b/tests/e2e/errors/def_label.hny new file mode 100644 index 00000000..3860e451 --- /dev/null +++ b/tests/e2e/errors/def_label.hny @@ -0,0 +1,2 @@ + +def: x = 3 diff --git a/tests/e2e/errors/empty.hny b/tests/e2e/errors/empty.hny new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/errors/eof_string.hny b/tests/e2e/errors/eof_string.hny new file mode 100644 index 00000000..cc80910f --- /dev/null +++ b/tests/e2e/errors/eof_string.hny @@ -0,0 +1,2 @@ + +v = "Hello World diff --git a/tests/e2e/errors/missing_indent.hny b/tests/e2e/errors/missing_indent.hny new file mode 100644 index 00000000..7fcfa775 --- /dev/null +++ b/tests/e2e/errors/missing_indent.hny @@ -0,0 +1,3 @@ + +def ok(): +pass diff --git a/tests/e2e/errors/unclosed_paren.hny b/tests/e2e/errors/unclosed_paren.hny new file mode 100644 index 00000000..422e7aa3 --- /dev/null +++ b/tests/e2e/errors/unclosed_paren.hny @@ -0,0 +1,2 @@ + +a, b = (1, 5, \ No newline at end of file diff --git a/tests/e2e/load_test_files.py b/tests/e2e/load_test_files.py new file mode 100644 index 00000000..68577faf --- /dev/null +++ b/tests/e2e/load_test_files.py @@ -0,0 +1,49 @@ +from datetime import timedelta +import pathlib +from typing import List, NamedTuple + + +class Params(NamedTuple): + filename: pathlib.Path + max_time: timedelta + modules: List[str] + constants: List[str] + +def load_dir(dir: pathlib.Path, modules=None, constants=None): + return [ + Params( + filename=f, + max_time=timedelta(seconds=3), + modules=modules or [], + constants=constants or [], + ) for f in dir.glob("*.hny") + ] + +_DIR = pathlib.Path(__file__).parent + +def load_public_harmony_files(): + code_dir = _DIR.parent.parent / "code" + return load_dir(code_dir) + +def load_failing_harmony_files(): + code_dir = _DIR / "errors" + return load_dir(code_dir) + +def load_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=42']) + +def load_failing_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=']) \ + + load_dir(code_dir, constants=['C=42', 'A=12']) # unused constants + +def load_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/math']) + +def load_failing_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/matt']) \ + + load_dir(code_dir, modules=['math=resources/math', + 'numpy=resources/numpy']) # unused modules diff --git a/tests/e2e/modules/base.hny b/tests/e2e/modules/base.hny new file mode 100644 index 00000000..d1cb7cbe --- /dev/null +++ b/tests/e2e/modules/base.hny @@ -0,0 +1,4 @@ + +import math + +print math.sin(10) diff --git a/tests/e2e/modules/resources/math.hny b/tests/e2e/modules/resources/math.hny new file mode 100644 index 00000000..0ab098dd --- /dev/null +++ b/tests/e2e/modules/resources/math.hny @@ -0,0 +1,3 @@ + +def sin(x): + result = "0.12423" diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py new file mode 100644 index 00000000..8ad24e59 --- /dev/null +++ b/tests/e2e/test_compilation.py @@ -0,0 +1,49 @@ +import unittest +from tests.e2e.load_test_files import * + +import time +from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection + +import harmony_model_checker.harmony.harmony as legacy_harmony +from harmony_model_checker.compile import do_compile + +class CompilationTestCase(unittest.TestCase): + def run_before_tests(self): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled + + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() + + def test_compilation_success(self): + params = load_public_harmony_files() \ + + load_constant_defined_harmony_files() \ + + load_module_defined_harmony_files() + for param in params: + f = str(param.filename) + self.run_before_tests() + with self.subTest(f"Success compilation test: {f}"): + start_time = time.perf_counter_ns() + do_compile(f, consts=param.constants, mods=param.modules, interface=None) + end_time = time.perf_counter_ns() + duration = end_time - start_time + self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9) + + def test_compilation_error(self): + params = load_failing_harmony_files() \ + + load_failing_constant_defined_harmony_files() \ + + load_failing_module_defined_harmony_files() + for param in params: + f = str(param.filename) + self.run_before_tests() + with self.subTest(f"Failure compilation test: {f}"): + start_time = time.perf_counter_ns() + self.assertRaises( + (HarmonyCompilerErrorCollection, HarmonyCompilerError), + lambda: do_compile(f, consts=param.constants, mods=param.modules, interface=None)) + end_time = time.perf_counter_ns() + duration = end_time - start_time + self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9) diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py new file mode 100644 index 00000000..1a044f11 --- /dev/null +++ b/tests/e2e/test_gen_html.py @@ -0,0 +1,67 @@ +import logging +import subprocess +import unittest +from harmony_model_checker.main import handle_hco +import harmony_model_checker.harmony.harmony as legacy_harmony + +from tests.e2e.load_test_files import * + +logger = logging.Logger(__file__) + +class MockNS: + B = None + noweb = True + const = None + mods = None + intf = None + module = None + cf = [] + suppress = False + +_HARMONY_SCRIPT = pathlib.Path(__file__).parent.parent.parent / "harmony" + +def _replace_ext(p: pathlib.Path, ext: str): + p_ext = p.suffix + return str(p)[:-len(p_ext)] + '.' + ext + +class GenHtmlTestCase(unittest.TestCase): + def run_before_tests(self): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled + + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() + + def test_gen_html(self): + params = load_public_harmony_files() \ + + load_constant_defined_harmony_files() \ + + load_module_defined_harmony_files() + mock_ns = MockNS() + for param in params: + output_files = { + "hfa": None, + "htm": _replace_ext(param.filename, 'htm'), + "hco": _replace_ext(param.filename, 'hco'), + "hvm": _replace_ext(param.filename, 'hvm'), + "hvb": _replace_ext(param.filename, 'hvb'), + "png": None, + "tla": None, + "gv": None + } + with self.subTest(): + try: + # If it takes longer than 3 seconds, just skip. + r = subprocess.run( + args=[_HARMONY_SCRIPT, + str(param.filename), '--noweb'] + + [('-c' + c) for c in param.constants] + + [('-m' + m) for m in param.modules], + timeout=3) + self.assertEqual(r.returncode, 0) + except subprocess.TimeoutExpired: + logger.warning("TimeoutExpired for file %s.", str(param.filename)) + continue + handle_hco(mock_ns, output_files) diff --git a/tests/harmony/test_ast.py b/tests/harmony/test_ast.py new file mode 100644 index 00000000..7d31a278 --- /dev/null +++ b/tests/harmony/test_ast.py @@ -0,0 +1,111 @@ +from unittest import TestCase +import unittest + +from harmony_model_checker.harmony.ast import * +from harmony_model_checker.harmony.code import Labeled_Op + +def create_token(value, file='test.hny', line=0, col=0): + return value, file, line, col + +class TestConstantAST(TestCase): + def create_constant_ast(self): + return [ + ConstantAST( + endtoken=create_token(const), + const=create_token(const), + ) for const in [12, True, "str"] + ] + + def test_is_constant(self): + for c in self.create_constant_ast(): + scope = Scope(None) + self.assertTrue(c.isConstant(scope)) + + def test_compile(self): + for c in self.create_constant_ast(): + scope = Scope(None) + code = Code() + c.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + + op: PushOp = lbled_op.op + self.assertIsInstance(op, PushOp) + self.assertEqual(op.constant, c.const) + + +class TestNameAST(TestCase): + def create_name_ast(self): + return [ + NameAST( + endtoken=create_token(name), + name=create_token(name), + ) for name in ['abc', 'foo', 'bar', 'harmony'] + ] + + def test_is_constant(self): + for n in self.create_name_ast(): + # default scope is global + scope = Scope(None) + self.assertFalse(n.isConstant(scope)) + + scope = Scope(None) + lexeme = n.name[0] + scope.names[lexeme] = ("constant", n.name) + self.assertTrue(n.isConstant(scope)) + + def test_compile(self): + for n in self.create_name_ast(): + lexeme = n.name[0] + + # test with global scope + scope = Scope(None) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: LoadOp = lbled_op.op + self.assertIsInstance(op, LoadOp) + self.assertEqual(op.name, n.name) + + # test as a local variable + scope = Scope(None) + scope.names[lexeme] = ("local-var", n.name) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: LoadVarOp = lbled_op.op + self.assertIsInstance(op, LoadVarOp) + self.assertEqual(op.v, n.name) + + # test as a constant + scope = Scope(None) + scope.names[lexeme] = ("constant", n.name) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: PushOp = lbled_op.op + self.assertIsInstance(op, PushOp) + self.assertEqual(op.constant, n.name) + + +class TestHarmonyAST(TestCase): + """Tests the creation and modification of Harmony AST classes + """ + def test_create(self): + pass + + def test_simple(self): + self.assertTrue(True) + pass + + +if __name__ == "__main__": + unittest.main()