Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add test-suite to run compiler and html generator #6

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,5 @@ book/pp.tex

# Auto-generated by Makefile's [gen] target
harmony_model_checker/__init__.py

.coverage
17 changes: 17 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,25 @@ upload-test: dist
upload: dist
twine upload dist/*

test-unit:
coverage run -m unittest discover tests/harmony

test-e2e:
coverage run -m unittest discover tests/e2e

test: test-unit test-e2e

clean:
# Harmony outputs in `code` directory
rm -f code/*.htm code/*.hvm code/*.hco code/*.png code/*.hfa code/*.tla code/*.gv *.html

# Harmony outputs in `modules` directory
(cd harmony_model_checker/modules; rm -f *.htm *.hvm *.hco *.png *.hfa *.tla *.gv *.html)

rm -rf compiler_integration_results.md compiler_integration_results/

# Package publication related outputs
rm -rf build/ dist/ harmony_model_checker.egg-info/

# Test coverage related outputs
rm -rf .coverage htmlcov
17 changes: 9 additions & 8 deletions code/BBsema.hny
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import synch;
from synch import Semaphore, P, V;

const NSLOTS = 2; # size of bounded buffer

buf = { x:() for x in {1..NSLOTS} };
b_in = 1;
b_out = 1;
l_in = Semaphore(1);
l_out = Semaphore(1);
n_full = Semaphore(0);
n_empty = Semaphore(NSLOTS);

def produce(item):
P(?n_empty);
P(?l_in);
Expand All @@ -18,10 +26,3 @@ def consume():
V(?l_out);
V(?n_empty);
;
buf = { x:() for x in {1..NSLOTS} };
b_in = 1;
b_out = 1;
l_in = Semaphore(1);
l_out = Semaphore(1);
n_full = Semaphore(0);
n_empty = Semaphore(NSLOTS);
2 changes: 1 addition & 1 deletion code/BBsematest.hny
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import BBsema;
from BBsema import produce, consume;

const NPRODS = 3; # number of producers
const NCONSS = 3; # number of consumers
Expand Down
4 changes: 2 additions & 2 deletions harmony_model_checker/harmony/ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -1270,9 +1270,9 @@ def compile(self, scope, code, stmt):
for var_or_cond in self.vars_and_conds:
if var_or_cond[0] == 'var':
(_, var, expr, tkn, endtkn, op) = var_or_cond
stmt = self.range(token, endtkn)
stmt = self.range(tkn, endtkn)
expr.compile(ns, code, stmt)
code.append(StoreVarOp(var), token, op, stmt=stmt)
code.append(StoreVarOp(var), tkn, op, stmt=stmt)
self.define(ns, var)
elif var_or_cond[0] == 'cond':
(_, cond, token, endtkn) = var_or_cond
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
antlr-denter==1.3.1
antlr4-python3-runtime==4.9.3
automata-lib==5.0.0
coverage==6.3.2
pydot==1.4.2
requests==2.27.1
twine==3.7.1
11 changes: 11 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Harmony Tests

Types of tests:

- Compilation/Performance Tests
- Unit Tests

## Compilation/Performance Tests

## Unit Tests

Empty file added tests/e2e/__init__.py
Empty file.
3 changes: 3 additions & 0 deletions tests/e2e/constants/base.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

const C = 23
print C
5 changes: 5 additions & 0 deletions tests/e2e/errors/bad_elif_stmt.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

if True:
pass
else if:
pass
2 changes: 2 additions & 0 deletions tests/e2e/errors/def_label.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

def: x = 3
Empty file added tests/e2e/errors/empty.hny
Empty file.
2 changes: 2 additions & 0 deletions tests/e2e/errors/eof_string.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

v = "Hello World
3 changes: 3 additions & 0 deletions tests/e2e/errors/missing_indent.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

def ok():
pass
2 changes: 2 additions & 0 deletions tests/e2e/errors/unclosed_paren.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

a, b = (1, 5,
49 changes: 49 additions & 0 deletions tests/e2e/load_test_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from datetime import timedelta
import pathlib
from typing import List, NamedTuple


class Params(NamedTuple):
filename: pathlib.Path
max_time: timedelta
modules: List[str]
constants: List[str]

def load_dir(dir: pathlib.Path, modules=None, constants=None):
return [
Params(
filename=f,
max_time=timedelta(seconds=3),
modules=modules or [],
constants=constants or [],
) for f in dir.glob("*.hny")
]

_DIR = pathlib.Path(__file__).parent

def load_public_harmony_files():
code_dir = _DIR.parent.parent / "code"
return load_dir(code_dir)

def load_failing_harmony_files():
code_dir = _DIR / "errors"
return load_dir(code_dir)

def load_constant_defined_harmony_files():
code_dir = _DIR / "constants"
return load_dir(code_dir, constants=['C=42'])

def load_failing_constant_defined_harmony_files():
code_dir = _DIR / "constants"
return load_dir(code_dir, constants=['C=']) \
+ load_dir(code_dir, constants=['C=42', 'A=12']) # unused constants

def load_module_defined_harmony_files():
code_dir = _DIR / "modules"
return load_dir(code_dir, modules=['math=resources/math'])

def load_failing_module_defined_harmony_files():
code_dir = _DIR / "modules"
return load_dir(code_dir, modules=['math=resources/matt']) \
+ load_dir(code_dir, modules=['math=resources/math',
'numpy=resources/numpy']) # unused modules
4 changes: 4 additions & 0 deletions tests/e2e/modules/base.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@

import math

print math.sin(10)
3 changes: 3 additions & 0 deletions tests/e2e/modules/resources/math.hny
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

def sin(x):
result = "0.12423"
49 changes: 49 additions & 0 deletions tests/e2e/test_compilation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import unittest
from tests.e2e.load_test_files import *

import time
from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection

import harmony_model_checker.harmony.harmony as legacy_harmony
from harmony_model_checker.compile import do_compile

class CompilationTestCase(unittest.TestCase):
def run_before_tests(self):
legacy_harmony.files.clear() # files that have been read already
legacy_harmony.modules.clear() # modules modified with -m
legacy_harmony.used_modules.clear() # modules modified and used
legacy_harmony.namestack.clear() # stack of module names being compiled

legacy_harmony.imported.clear()
legacy_harmony.constants.clear()
legacy_harmony.used_constants.clear()

def test_compilation_success(self):
params = load_public_harmony_files() \
+ load_constant_defined_harmony_files() \
+ load_module_defined_harmony_files()
for param in params:
f = str(param.filename)
self.run_before_tests()
with self.subTest(f"Success compilation test: {f}"):
start_time = time.perf_counter_ns()
do_compile(f, consts=param.constants, mods=param.modules, interface=None)
end_time = time.perf_counter_ns()
duration = end_time - start_time
self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9)

def test_compilation_error(self):
params = load_failing_harmony_files() \
+ load_failing_constant_defined_harmony_files() \
+ load_failing_module_defined_harmony_files()
for param in params:
f = str(param.filename)
self.run_before_tests()
with self.subTest(f"Failure compilation test: {f}"):
start_time = time.perf_counter_ns()
self.assertRaises(
(HarmonyCompilerErrorCollection, HarmonyCompilerError),
lambda: do_compile(f, consts=param.constants, mods=param.modules, interface=None))
end_time = time.perf_counter_ns()
duration = end_time - start_time
self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9)
67 changes: 67 additions & 0 deletions tests/e2e/test_gen_html.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import logging
import subprocess
import unittest
from harmony_model_checker.main import handle_hco
import harmony_model_checker.harmony.harmony as legacy_harmony

from tests.e2e.load_test_files import *

logger = logging.Logger(__file__)

class MockNS:
B = None
noweb = True
const = None
mods = None
intf = None
module = None
cf = []
suppress = False

_HARMONY_SCRIPT = pathlib.Path(__file__).parent.parent.parent / "harmony"

def _replace_ext(p: pathlib.Path, ext: str):
p_ext = p.suffix
return str(p)[:-len(p_ext)] + '.' + ext

class GenHtmlTestCase(unittest.TestCase):
def run_before_tests(self):
legacy_harmony.files.clear() # files that have been read already
legacy_harmony.modules.clear() # modules modified with -m
legacy_harmony.used_modules.clear() # modules modified and used
legacy_harmony.namestack.clear() # stack of module names being compiled

legacy_harmony.imported.clear()
legacy_harmony.constants.clear()
legacy_harmony.used_constants.clear()

def test_gen_html(self):
params = load_public_harmony_files() \
+ load_constant_defined_harmony_files() \
+ load_module_defined_harmony_files()
mock_ns = MockNS()
for param in params:
output_files = {
"hfa": None,
"htm": _replace_ext(param.filename, 'htm'),
"hco": _replace_ext(param.filename, 'hco'),
"hvm": _replace_ext(param.filename, 'hvm'),
"hvb": _replace_ext(param.filename, 'hvb'),
"png": None,
"tla": None,
"gv": None
}
with self.subTest():
try:
# If it takes longer than 3 seconds, just skip.
r = subprocess.run(
args=[_HARMONY_SCRIPT,
str(param.filename), '--noweb'] +
[('-c' + c) for c in param.constants] +
[('-m' + m) for m in param.modules],
timeout=3)
self.assertEqual(r.returncode, 0)
except subprocess.TimeoutExpired:
logger.warning("TimeoutExpired for file %s.", str(param.filename))
continue
handle_hco(mock_ns, output_files)
Loading