diff --git a/README.md b/README.md index e4d0a96e..a56c1717 100644 --- a/README.md +++ b/README.md @@ -26,5 +26,6 @@ Change log: - v1.2.0 Updates to auto testing and minors bug fixes. Version changes for HELICS 3.3, GridLAB-D 5.0. Reorganize environment and updated documentation. Add more loadshed N3 examples. - v1.2.1 Fixed python shell files, fixed complex python to use helics complex. - v1.2.2 Fixed the installation for Ubuntu 22.04. -- v1.3.0 Refactor the TESP PyPI api. Upgrade all models(GridLAB-D, EnergyPlus, NS3) to work with HELICS 3.4. Add modifier.py for GridLAB-D models -- v1.3.2 Updated model and modifier for GridLAB-D models, added readme for GLM modifier and Store examples +- v1.3.0 Refactor the TESP PyPI api. Upgrade all models(GridLAB-D, EnergyPlus, NS3) to work with HELICS 3.4. Add modifier.py for GridLAB-D models. +- v1.3.2 Updated model and modifier for GridLAB-D models, added readme for GLM modifier and Store examples. +- v1.3.3 Add tesp_component download in tesp_support pypi, change directory structure. Add dockerfile for each module getting ready for dockerize COSU Simulations \ No newline at end of file diff --git a/doc/references/tesp_support.api.rst b/doc/references/tesp_support.api.rst index 9fe583a0..0eede4eb 100644 --- a/doc/references/tesp_support.api.rst +++ b/doc/references/tesp_support.api.rst @@ -81,18 +81,18 @@ tesp\_support.api.metrics\_collector module :undoc-members: :show-inheritance: -tesp\_support.api.model module ------------------------------- +tesp\_support.api.model\_GLM module +----------------------------------- -.. automodule:: tesp_support.api.model +.. automodule:: tesp_support.api.model_GLM :members: :undoc-members: :show-inheritance: -tesp\_support.api.modifier module ---------------------------------- +tesp\_support.api.modify\_GLM module +------------------------------------ -.. automodule:: tesp_support.api.modifier +.. automodule:: tesp_support.api.modify_GLM :members: :undoc-members: :show-inheritance: diff --git a/examples/autotest.py b/examples/autotest.py index 98a4a4bd..a6a60e94 100644 --- a/examples/autotest.py +++ b/examples/autotest.py @@ -188,6 +188,8 @@ def combine_feeders_test(): if sys.argv[1] == "FNCS": b_helics = False + # tr.run_test = tr.run_docker_test + tr.init_tests() tesp_path = os.path.expandvars('$TESPDIR/examples') os.chdir(tesp_path) diff --git a/install/Docker/TESP-Docker-Inputs/runFNCS-TESP-Container-Mac.sh b/install/Docker/TESP-Docker-Inputs/runFNCS-TESP-Container-Mac.sh index 72d560e0..950248df 100644 --- a/install/Docker/TESP-Docker-Inputs/runFNCS-TESP-Container-Mac.sh +++ b/install/Docker/TESP-Docker-Inputs/runFNCS-TESP-Container-Mac.sh @@ -1,7 +1,7 @@ +#!/bin/bash # Copyright (c) 2021-2023 Battelle Memorial Institute # file: runFNCS-TESP-Container-Mac.sh -#!/bin/bash clear # TESP_REP="laurmarinovici/tesp" @@ -60,7 +60,7 @@ echo "===== Setting up GridLAB-D paths and folders." docker container exec ${TESP_CONT} /bin/bash -c 'if test -e ${GLD_INSTALL}/bin/outputFiles; then rmdir ${GLD_INSTALL}/bin/outputFiles; mkdir ${GLD_INSTALL}/bin/outputFiles; else mkdir ${GLD_INSTALL}/bin/outputFiles; fi' # =================== Energy Plus settings ========================================================= -echo "===== Settting up Energy Plus paths and folders." +echo "===== Setting up Energy Plus paths and folders." docker container exec ${TESP_CONT} /bin/bash -c 'if test -e ${EPLUS_INSTALL}/outputFiles; then rmdir ${EPLUS_INSTALL}/outputFiles; mkdir ${EPLUS_INSTALL}/outputFiles; else mkdir ${EPLUS_INSTALL}/outputFiles; fi' # =================== Energy Plus JSON settings ========================================================= diff --git a/scripts/build/stamp.sh b/scripts/build/stamp.sh index a373f318..baa54e24 100755 --- a/scripts/build/stamp.sh +++ b/scripts/build/stamp.sh @@ -4,7 +4,7 @@ if [[ -z ${INSTDIR} ]]; then . "${HOME}/tespEnv" fi -ver="1.3.2" +ver="1.3.3" echo echo "Stamping TESP $ver, if you want to change the version, edit this file." diff --git a/scripts/build/tesp_pypi.id b/scripts/build/tesp_pypi.id index 35b4963e..51780896 100644 --- a/scripts/build/tesp_pypi.id +++ b/scripts/build/tesp_pypi.id @@ -1,131 +1,230 @@ -Package Version Editable project location ------------------------------ ------------ ------------------------------------------------ -alabaster 0.7.13 -aniso8601 9.0.1 -attrs 23.1.0 -Babel 2.13.0 -blinker 1.6.3 -blosc2 2.0.0 -build 1.0.3 -certifi 2023.7.22 -cffi 1.16.0 -charset-normalizer 3.3.0 -click 8.1.7 -colorama 0.4.6 -commonmark 0.9.1 -contourpy 1.1.1 -cryptography 41.0.4 -cycler 0.12.1 -Cython 3.0.3 -docutils 0.18.1 -et-xmlfile 1.1.0 -Flask 3.0.0 -Flask-Cors 4.0.0 -Flask-RESTful 0.3.10 -fonttools 4.43.1 -future 0.18.3 -ghp-import 2.1.0 -greenlet 3.0.0 -h5py 3.9.0 -helics 3.4.0 -idna 3.4 -imagesize 1.4.1 -importlib-metadata 6.8.0 -importlib-resources 6.1.0 -itsdangerous 2.1.2 -jaraco.classes 3.3.0 -jeepney 0.8.0 -Jinja2 3.1.2 -joblib 1.3.2 -JsonForm 0.0.2 -jsonpointer 2.4 -jsonschema 4.19.1 -jsonschema-specifications 2023.7.1 -JsonSir 0.0.2 -keyring 24.2.0 -kiwisolver 1.4.5 -latexcodec 2.0.1 -markdown-it-py 3.0.0 -MarkupSafe 2.1.3 -matplotlib 3.7.3 -mdurl 0.1.2 -more-itertools 10.1.0 -msgpack 1.0.7 -networkx 3.1 -nh3 0.2.14 -nose 1.3.7 -NREL-PySAM 4.2.0 -numexpr 2.8.6 -numpy 1.24.4 -openpyxl 3.1.2 -packaging 23.2 -pandas 2.0.3 -Pillow 10.0.1 -pip 23.2.1 -pkginfo 1.9.6 -pkgutil_resolve_name 1.3.10 -plotly 5.17.0 -ply 3.11 -psst 0.1.1a0 /home/tesp/tesp/repository/AMES-V5.0/psst -PuLP 2.7.0 -py-cpuinfo 9.0.0 -pybtex 0.24.0 -pybtex-docutils 1.0.3 -pycparser 2.21 -PyGLM 2.7.0 -Pygments 2.16.1 -Pyomo 6.5.0 -pyparsing 3.1.1 -PYPOWER 5.1.16 -pyproject_hooks 1.0.0 -python-dateutil 2.8.2 -python-dotenv 1.0.0 -Python-EasyConfig 0.1.7 -pytz 2023.3.post1 -PyUtilib 6.0.0 -PyYAML 6.0.1 -readme-renderer 42.0 -recommonmark 0.7.1 -referencing 0.30.2 -requests 2.31.0 -requests-toolbelt 1.0.0 -Resource 0.2.1 -rfc3986 2.0.0 -rich 13.6.0 -rpds-py 0.10.4 -scikit-learn 1.3.1 -scipy 1.10.1 -seaborn 0.13.0 -SecretStorage 3.3.3 -setuptools 68.2.0 -six 1.16.0 -snowballstemmer 2.2.0 -Sphinx 7.1.2 -sphinx-jsonschema 1.19.1 -sphinx-rtd-theme 1.3.0 -sphinxcontrib-applehelp 1.0.4 -sphinxcontrib-bibtex 2.5.0 -sphinxcontrib-devhelp 1.0.2 -sphinxcontrib-fulltoc 1.2.0 -sphinxcontrib-htmlhelp 2.0.1 -sphinxcontrib-jquery 4.1 -sphinxcontrib-jsdemo 0.1.4 -sphinxcontrib-jsmath 1.0.1 -sphinxcontrib-qthelp 1.0.3 -sphinxcontrib-serializinghtml 1.1.5 -SQLAlchemy 2.0.21 -strip-hints 0.1.10 -tables 3.8.0 -tenacity 8.2.3 -tesp-support 1.3.1 /home/tesp/tesp/repository/tesp/src/tesp_support -threadpoolctl 3.2.0 -tomli 2.0.1 -twine 4.0.2 -typing_extensions 4.8.0 -tzdata 2023.3 -urllib3 2.0.6 -Werkzeug 3.0.0 -wheel 0.41.2 -xarray 2023.1.0 -xlrd 2.0.1 -zipp 3.17.0 +Package Version Editable project location +---------------------------------------- ------------ ------------------------------------------------ +a2wsgi 1.8.0 +aiohttp 3.8.6 +aiosignal 1.3.1 +alabaster 0.7.13 +alembic 1.12.1 +aniso8601 9.0.1 +annotated-types 0.6.0 +anyio 4.0.0 +apache-airflow 2.7.3 +apache-airflow-providers-common-sql 1.8.0 +apache-airflow-providers-ftp 3.6.0 +apache-airflow-providers-http 4.6.0 +apache-airflow-providers-imap 3.4.0 +apache-airflow-providers-sqlite 3.5.0 +apispec 6.3.0 +argcomplete 3.1.3 +asgiref 3.7.2 +async-timeout 4.0.3 +attrs 23.1.0 +Babel 2.13.1 +backoff 1.10.0 +blinker 1.6.3 +blosc2 2.0.0 +build 1.0.3 +cachelib 0.9.0 +cattrs 23.1.2 +certifi 2023.7.22 +cffi 1.16.0 +charset-normalizer 3.3.2 +click 8.1.7 +clickclick 20.10.2 +colorama 0.4.6 +colorlog 4.8.0 +commonmark 0.9.1 +ConfigUpdater 3.1.1 +connexion 2.14.2 +contourpy 1.1.1 +cron-descriptor 1.4.0 +croniter 2.0.1 +cryptography 41.0.5 +cycler 0.12.1 +Cython 3.0.3 +Deprecated 1.2.14 +dill 0.3.1.1 +dnspython 2.4.2 +docutils 0.18.1 +email-validator 1.3.1 +et-xmlfile 1.1.0 +exceptiongroup 1.1.3 +Flask 2.2.5 +Flask-AppBuilder 4.3.6 +Flask-Babel 2.0.0 +Flask-Caching 2.1.0 +Flask-Cors 4.0.0 +Flask-JWT-Extended 4.5.3 +Flask-Limiter 3.5.0 +Flask-Login 0.6.3 +Flask-RESTful 0.3.10 +Flask-Session 0.5.0 +Flask-SQLAlchemy 2.5.1 +Flask-WTF 1.2.1 +fonttools 4.43.1 +frozenlist 1.4.0 +future 0.18.3 +ghp-import 2.1.0 +google-re2 1.1 +googleapis-common-protos 1.61.0 +graphviz 0.20.1 +greenlet 3.0.1 +grpcio 1.59.2 +gunicorn 21.2.0 +h11 0.14.0 +h5py 3.9.0 +helics 3.4.0 +httpcore 0.16.3 +httpx 0.23.3 +idna 3.4 +imagesize 1.4.1 +importlib-metadata 6.8.0 +importlib-resources 6.1.0 +inflection 0.5.1 +itsdangerous 2.1.2 +jaraco.classes 3.3.0 +jeepney 0.8.0 +Jinja2 3.1.2 +joblib 1.3.2 +JsonForm 0.0.2 +jsonpointer 2.4 +jsonschema 4.19.2 +jsonschema-specifications 2023.7.1 +JsonSir 0.0.2 +keyring 24.2.0 +kiwisolver 1.4.5 +latexcodec 2.0.1 +lazy-object-proxy 1.9.0 +limits 3.6.0 +linkify-it-py 2.0.2 +lockfile 0.12.2 +Mako 1.2.4 +Markdown 3.5.1 +markdown-it-py 3.0.0 +MarkupSafe 2.1.3 +marshmallow 3.20.1 +marshmallow-oneofschema 3.0.1 +marshmallow-sqlalchemy 0.26.1 +matplotlib 3.7.3 +mdit-py-plugins 0.4.0 +mdurl 0.1.2 +more-itertools 10.1.0 +msgpack 1.0.7 +multidict 6.0.4 +networkx 3.1 +nh3 0.2.14 +nose 1.3.7 +NREL-PySAM 4.2.0 +numexpr 2.8.6 +numpy 1.24.4 +openpyxl 3.1.2 +opentelemetry-api 1.20.0 +opentelemetry-exporter-otlp 1.20.0 +opentelemetry-exporter-otlp-proto-common 1.20.0 +opentelemetry-exporter-otlp-proto-grpc 1.20.0 +opentelemetry-exporter-otlp-proto-http 1.20.0 +opentelemetry-proto 1.20.0 +opentelemetry-sdk 1.20.0 +opentelemetry-semantic-conventions 0.41b0 +ordered-set 4.1.0 +packaging 23.2 +pandas 2.0.3 +pathspec 0.11.2 +pendulum 2.1.2 +Pillow 10.0.1 +pip 23.3.1 +pkginfo 1.9.6 +pkgutil_resolve_name 1.3.10 +plotly 5.17.0 +pluggy 1.3.0 +ply 3.11 +prison 0.2.1 +protobuf 4.24.4 +psst 0.1.1a0 /home/tesp/tesp/repository/AMES-V5.0/psst +psutil 5.9.6 +PuLP 2.7.0 +py-cpuinfo 9.0.0 +pybtex 0.24.0 +pybtex-docutils 1.0.3 +pycparser 2.21 +pydantic 2.4.2 +pydantic_core 2.10.1 +PyGLM 2.7.0 +Pygments 2.16.1 +PyJWT 2.8.0 +Pyomo 6.5.0 +pyparsing 3.1.1 +PYPOWER 5.1.16 +pyproject_hooks 1.0.0 +python-daemon 3.0.1 +python-dateutil 2.8.2 +python-dotenv 1.0.0 +Python-EasyConfig 0.1.7 +python-multipart 0.0.6 +python-nvd3 0.15.0 +python-slugify 8.0.1 +pytz 2023.3.post1 +pytzdata 2020.1 +PyUtilib 6.0.0 +PyYAML 6.0.1 +readme-renderer 42.0 +recommonmark 0.7.1 +referencing 0.30.2 +requests 2.31.0 +requests-toolbelt 1.0.0 +Resource 0.2.1 +rfc3339-validator 0.1.4 +rfc3986 1.5.0 +rich 13.6.0 +rich-argparse 1.4.0 +rpds-py 0.10.6 +scikit-learn 1.3.1 +scipy 1.10.1 +seaborn 0.13.0 +SecretStorage 3.3.3 +setproctitle 1.3.3 +setuptools 68.2.0 +six 1.16.0 +sniffio 1.3.0 +snowballstemmer 2.2.0 +Sphinx 7.1.2 +sphinx-jsonschema 1.19.1 +sphinx-rtd-theme 1.3.0 +sphinxcontrib-applehelp 1.0.4 +sphinxcontrib-bibtex 2.5.0 +sphinxcontrib-devhelp 1.0.2 +sphinxcontrib-fulltoc 1.2.0 +sphinxcontrib-htmlhelp 2.0.1 +sphinxcontrib-jquery 4.1 +sphinxcontrib-jsdemo 0.1.4 +sphinxcontrib-jsmath 1.0.1 +sphinxcontrib-qthelp 1.0.3 +sphinxcontrib-serializinghtml 1.1.5 +SQLAlchemy 1.4.50 +SQLAlchemy-JSONField 1.0.1.post0 +SQLAlchemy-Utils 0.41.1 +sqlparse 0.4.4 +starlette 0.31.1 +strip-hints 0.1.10 +tables 3.8.0 +tabulate 0.9.0 +tenacity 8.2.3 +termcolor 2.3.0 +tesp-support 1.3.2 /home/tesp/tesp/repository/tesp/src/tesp_support +text-unidecode 1.3 +threadpoolctl 3.2.0 +tomli 2.0.1 +twine 4.0.2 +typing_extensions 4.8.0 +tzdata 2023.3 +uc-micro-py 1.0.2 +unicodecsv 0.14.1 +urllib3 2.0.7 +Werkzeug 2.2.3 +wheel 0.41.2 +wrapt 1.15.0 +WTForms 3.0.1 +xarray 2023.1.0 +xlrd 2.0.1 +yarl 1.9.2 +zipp 3.17.0 diff --git a/scripts/build/versions.sh b/scripts/build/versions.sh index adb35931..d7f58047 100755 --- a/scripts/build/versions.sh +++ b/scripts/build/versions.sh @@ -27,7 +27,7 @@ energyplus --version message="NS-3 not installed" for file in "${INSTDIR}"/bin/ns3-dev-* ; do lst=( ${file// / } ) - for a in "${lst}" ; do + for a in ${lst} ; do if [[ -f "${a}" ]]; then message="NS-3 installed" break diff --git a/scripts/version b/scripts/version index 1892b926..31e5c843 100644 --- a/scripts/version +++ b/scripts/version @@ -1 +1 @@ -1.3.2 +1.3.3 diff --git a/src/tesp_support/CHANGELOG.rst b/src/tesp_support/CHANGELOG.rst index e6baceea..cd4e2b1b 100644 --- a/src/tesp_support/CHANGELOG.rst +++ b/src/tesp_support/CHANGELOG.rst @@ -84,3 +84,12 @@ Version 1.3.1 (2023-10-3) Version 1.3.2 (2023-10-27) -------------------------- * Another fix for model and modifier for GridLAB-D models + +Version 1.3.3 (2023-12-4) +-------------------------- +* Add tesp_component download in tesp_support pypi, add schedule in model_GLM.py +* Minor changes to path for RECS system feeder +* Restructured, new RECS parameters, updated to include income, new hvac setpoint RECS data, teleworking. +* Fix postposing for DSOT +* Dockerfile for each module getting ready for dockerize COSU Simulations +* Change directory structure \ No newline at end of file diff --git a/src/tesp_support/tesp_support/api/test_runner.py b/src/tesp_support/tesp_support/api/test_runner.py index bcf176fc..03fadc3c 100644 --- a/src/tesp_support/tesp_support/api/test_runner.py +++ b/src/tesp_support/tesp_support/api/test_runner.py @@ -38,12 +38,22 @@ def start_test(case_name=None): def process_line(line, local_vars): # print('@@@@ input line to execute:', line) - foreground = line.replace(' &)', ')').replace(' &>', ' >') exports = '' for var in local_vars: exports = exports + 'export ' + var['key'] + '=' + var['val'] + ' && ' - # print(' line transformed to:', exports + foreground) - return exports + foreground + command = line.replace(' &)', ')').replace(' &>', ' >') + # print(' line transformed to:', exports + command) + return exports + command + + +def docker_line(line, local_vars): + # print('@@@@ input line to execute:', line) + exports = '' + for var in local_vars: + exports = exports + ' ' + var['key'] + ': "' + var['val'] + '"\n' + command = line.replace(' &)', '').replace(' &>', ' >').replace('(', '') + # print('docker line transformed to:', exports + command) + return exports, command def exec_test(file_name, case_name=None): @@ -112,13 +122,35 @@ def run_test(file_name, case_name=None): print('== Done: ', case_name, flush=True) +def services(name, image, env, cnt, outfile, depends=None): + outfile.write(" " + name + ":\n") + outfile.write(" image: \"" + image + "\"\n") + # outfile.write(" container_name: ${CONTAINER_NAME:" + '-jupyter_notebook' + "}") + if env[0] != '': + outfile.write(" environment:\n") + outfile.write(env[0]) + outfile.write(" working_dir: /home/worker/case\n") + outfile.write(" volumes:\n") + outfile.write(" - .:/home/worker\n") + outfile.write(" - ../../../data:/home/worker/tesp/data\n") + if depends is not None: + outfile.write(" depends_on:\n") + outfile.write(" - " + depends + "\n") + outfile.write(" networks:\n") + outfile.write(" cu_net:\n") + outfile.write(" ipv4_address: 10.5.0." + str(cnt) + "\n") + outfile.write(" command: sh -c \"" + env[1] + "\"\n") + + def run_docker_test(file_name, case_name=None): t_start = time.time() + cnt = 2 local_vars = [] fp = open(file_name, 'r') - p_list = [] - p_FNCS_broker = None - p_HELICS_broker = None + op = open(file_name.replace("sh", "yaml"), 'w') + op.write('version: "3.8"\n') + op.write('services:\n') + p_broker = None print('\n== Run: ', case_name, flush=True) for ln in fp: line = ln.rstrip('\n') @@ -135,25 +167,54 @@ def run_docker_test(file_name, case_name=None): line.startswith('gridlabd') or line.startswith('TMY3toTMY2_ansi'): jc = subprocess.Popen(process_line(line, local_vars), shell=True) jc.wait() - elif 'fncs_broker' in line: - print('==== Fncs Broker Start in\n ' + os.getcwd(), flush=True) - p_FNCS_broker = subprocess.Popen(process_line(line, local_vars), shell=True) - - elif 'helics_broker' in line: - print('==== Helics Broker Start in\n ' + os.getcwd(), flush=True) - p_HELICS_broker = subprocess.Popen(process_line(line, local_vars), shell=True) - else: # exec - pother = subprocess.Popen(process_line(line, local_vars), shell=True) - p_list.append(pother) + else: + if "fncs_broker" in line: + print('==== Fncs Broker Start in\n ' + os.getcwd(), flush=True) + p_broker = "fncs" + name = "fncs" + image = "tesp-fncs:latest" + elif "helics_broker" in line: + print('==== Helics Broker Start in\n ' + os.getcwd(), flush=True) + p_broker = "helics" + line = line.replace(' -f', ' --ipv4 -f') + name = "helics" + image = "tesp-helics:latest" + elif " python" in line or " java" in line: + name = "tespapi_" + str(cnt) + image = "tesp-tespapi:latest" + elif "gridlabd " in line: + name = "gridlabd_" + str(cnt) + image = "tesp-gridlabd:latest" + elif "energyplus " in line or "eplus_agent" in line: + name = "eplus_" + str(cnt) + image = "tesp-eplus:latest" + elif "helics_recorder " in line or "helics_player" in line: + name = "helics_" + str(cnt) + image = "tesp-helics:latest" + else: + # The master build, heavyweight docker + name = "build_" + str(cnt) + image = "tesp-build:latest" + cnt = cnt + 1 + services(name, image, docker_line(line, local_vars), cnt, op, p_broker) + + op.write('networks:\n') + op.write(' cu_net:\n') + op.write(' driver: bridge\n') + op.write(' ipam:\n') + op.write(' config:\n') + op.write(' - subnet: 10.5.0.0/16\n') + op.write(' gateway: 10.5.0.1\n') + op.close() fp.close() - if p_FNCS_broker is not None: - p_FNCS_broker.wait() - print('==== Fncs Broker Exit in\n ' + os.getcwd(), flush=True) - if p_HELICS_broker is not None: - p_HELICS_broker.wait() - print('==== Helics Broker Exit in\n ' + os.getcwd(), flush=True) - for p in p_list: - p.wait() + + if p_broker is not None: + # ps = subprocess.Popen(process_line("docker-compose -f run.yaml up -d", local_vars), shell=True) + # ps.wait() + print('==== Broker Exit in\n ' + os.getcwd(), flush=True) + # todo launch docker and wait + # for p in p_list: + # p.wait() t_end = time.time() if b_reporting: diff --git a/src/tesp_support/version b/src/tesp_support/version index 1892b926..31e5c843 100644 --- a/src/tesp_support/version +++ b/src/tesp_support/version @@ -1 +1 @@ -1.3.2 +1.3.3