diff --git a/Dockerfile b/Dockerfile index b6a48d49..f0a170b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG CKAN_VERSION USER root -RUN apt-get update && apt-get install -y postgresql-client openjdk-17-jre geos-dev +RUN apt-get update && apt-get install -y postgresql-client openjdk-17-jre libgeos-dev # Download Saxon jar for FGDC2ISO transform (geodatagov) ARG saxon_ver=9.9.1-7 diff --git a/ckanext/geodatagov/tests/conftest.py b/ckanext/geodatagov/tests/conftest.py new file mode 100644 index 00000000..bad5cb32 --- /dev/null +++ b/ckanext/geodatagov/tests/conftest.py @@ -0,0 +1,11 @@ +import pytest + +import utils + +@pytest.fixture(scope="session", autouse=True) +def run_once_for_all_tests(): + utils.simple_http_server() + +@pytest.fixture(scope="class", autouse=True) +def run_for_every_test_class(): + utils.reset_db_and_solr() \ No newline at end of file diff --git a/ckanext/geodatagov/tests/data-samples/waf-trim-tags/bad-tags.xml b/ckanext/geodatagov/tests/data-samples/waf-trim-tags/bad-tags.xml index c802211f..18b29ea7 100644 --- a/ckanext/geodatagov/tests/data-samples/waf-trim-tags/bad-tags.xml +++ b/ckanext/geodatagov/tests/data-samples/waf-trim-tags/bad-tags.xml @@ -2259,7 +2259,7 @@ - gov.noaa.class:CW_REGION + gov.noaa.class:CW_REGION1 diff --git a/ckanext/geodatagov/tests/data-samples/waf1/nodc_0099041_cw_region_meta_v2013.xml b/ckanext/geodatagov/tests/data-samples/waf1/nodc_0099041_cw_region_meta_v2013.xml index a23f8574..1524e6d5 100644 --- a/ckanext/geodatagov/tests/data-samples/waf1/nodc_0099041_cw_region_meta_v2013.xml +++ b/ckanext/geodatagov/tests/data-samples/waf1/nodc_0099041_cw_region_meta_v2013.xml @@ -1,6 +1,6 @@ - gov.noaa.class:CW_REGION + gov.noaa.class:CW_REGION1 eng @@ -3360,7 +3360,7 @@ - gov.noaa.class:CW_REGION + gov.noaa.class:CW_REGION1 diff --git a/ckanext/geodatagov/tests/mock_static_file_server.py b/ckanext/geodatagov/tests/mock_static_file_server.py deleted file mode 100644 index 6392056f..00000000 --- a/ckanext/geodatagov/tests/mock_static_file_server.py +++ /dev/null @@ -1,39 +0,0 @@ - -import http.server -import logging -import socketserver -from threading import Thread -import os -log = logging.getLogger(__name__) - - -PORT = 8999 - - -def serve(port=PORT): - '''Serves test XML files over HTTP''' - - # Make sure we serve from the tests' XML directory - os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'data-samples')) - - Handler = http.server.SimpleHTTPRequestHandler - - class TestServer(socketserver.TCPServer): - allow_reuse_address = True - - skip_connection = False - try: - httpd = TestServer(("", port), Handler) - except Exception as e: - print('Serve error {}'.format(e)) - skip_connection = True - - if skip_connection is False: - info = 'Serving test HTTP server at port', port - print(info) - log.info(info) - - httpd_thread = Thread(target=httpd.serve_forever) - httpd_thread.setDaemon(True) - httpd_thread.start() diff --git a/ckanext/geodatagov/tests/test_category_tags.py b/ckanext/geodatagov/tests/test_category_tags.py index 6cd48274..82029fc6 100644 --- a/ckanext/geodatagov/tests/test_category_tags.py +++ b/ckanext/geodatagov/tests/test_category_tags.py @@ -1,6 +1,7 @@ import logging import json import pytest + from ckan import plugins as p from ckan.tests import factories diff --git a/ckanext/geodatagov/tests/test_datajson.py b/ckanext/geodatagov/tests/test_datajson.py index bdaeb2b1..f276d0e6 100644 --- a/ckanext/geodatagov/tests/test_datajson.py +++ b/ckanext/geodatagov/tests/test_datajson.py @@ -1,45 +1,23 @@ import json -import os import pytest +import logging -from ckan.tests.helpers import reset_db from ckan.tests.factories import Organization from ckan import model -from ckan.model.meta import Session, metadata from factories import (DataJsonHarvestSourceObj, HarvestJobObj) import ckanext.harvest.model as harvest_model from ckanext.datajson.harvester_datajson import DataJsonHarvester -import mock_static_file_server -import logging +from utils import PORT, populate_locations_table log = logging.getLogger(__name__) @pytest.mark.usefixtures("with_plugins") class TestDataJsonHarvester(object): - @classmethod - def setup_class(cls): - log.info('Starting mock http server') - mock_static_file_server.serve(port=8996) - def setup_method(self): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - # echo "Downloading locations table" - os.system("wget https://github.com/GSA/datagov-deploy/raw/71936f004be1882a506362670b82c710c64ef796/" - "ansible/roles/software/ec2/ansible/files/locations.sql.gz -O /tmp/locations.sql.gz") - # echo "Creating locations table" - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'DROP TABLE IF EXISTS locations;'") - os.system("gunzip -c /tmp/locations.sql.gz | PGPASSWORD=ckan psql -h db -U ckan -d ckan -v ON_ERROR_STOP=1") - # echo "Cleaning" - os.system("rm -f /tmp/locations.sql.gz") - # os.system("ckan -c test.ini db upgrade -p harvest") - metadata.create_all(bind=Session.bind) + populate_locations_table() def run_gather(self, url): source = DataJsonHarvestSourceObj(url=url, owner_org=self.organization['id']) @@ -103,7 +81,7 @@ def test_sample5_data(self): self.organization = Organization() # testing with data from https://www.consumerfinance.gov/data.json - url = 'http://127.0.0.1:8996/sample5_data.json' + url = f'http://127.0.0.1:{PORT}/sample5_data.json' obj_ids = self.run_gather(url=url) assert len(obj_ids) == 2 self.run_fetch() diff --git a/ckanext/geodatagov/tests/test_db_solr_sync.py b/ckanext/geodatagov/tests/test_db_solr_sync.py index 2ddaf9f2..54b0f30b 100644 --- a/ckanext/geodatagov/tests/test_db_solr_sync.py +++ b/ckanext/geodatagov/tests/test_db_solr_sync.py @@ -1,6 +1,5 @@ import json import logging -import os import pytest from ckan.common import config @@ -8,9 +7,7 @@ import ckan.model as model import ckan.lib.search as search from ckan.tests import factories -from ckan.tests.helpers import reset_db from click.testing import CliRunner -from ckan.model.meta import Session, metadata from ckanext.harvest.model import HarvestObject from ckanext.harvest.tests import factories as harvest_factories @@ -23,16 +20,6 @@ @pytest.mark.usefixtures("with_plugins") class TestSolrDBSync(object): - @classmethod - def setup_class(cls): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - metadata.create_all(bind=Session.bind) - - search.clear_all() def create_datasets(self): diff --git a/ckanext/geodatagov/tests/test_fix_packages.py b/ckanext/geodatagov/tests/test_fix_packages.py index f5c1949e..4a3944d6 100644 --- a/ckanext/geodatagov/tests/test_fix_packages.py +++ b/ckanext/geodatagov/tests/test_fix_packages.py @@ -1,10 +1,7 @@ import pytest import logging -import os -from ckan.tests.helpers import reset_db from ckan.tests import factories -from ckan.model.meta import Session, metadata log = logging.getLogger(__name__) @@ -14,12 +11,6 @@ class TestFixPkg(object): @classmethod def setup_class(cls): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - metadata.create_all(bind=Session.bind) cls.organization = factories.Organization() def test_fix_tags(self): diff --git a/ckanext/geodatagov/tests/test_fix_spatial.py b/ckanext/geodatagov/tests/test_fix_spatial.py index 41afd28b..a9ebaec9 100644 --- a/ckanext/geodatagov/tests/test_fix_spatial.py +++ b/ckanext/geodatagov/tests/test_fix_spatial.py @@ -11,9 +11,6 @@ class TestSpatialField(object): @classmethod def setup_class(cls): - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - helpers.reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") # echo "Downloading locations table" os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'DROP TABLE IF EXISTS locations;'") os.system("wget https://github.com/GSA/datagov-deploy/raw/71936f004be1882a506362670b82c710c64ef796/" @@ -23,8 +20,6 @@ def setup_class(cls): # echo "Cleaning" os.system("rm -f /tmp/locations.sql.gz") - metadata.create_all(bind=Session.bind) - cls.user = factories.Sysadmin(name='spatial_user') def test_numeric_spatial_transformation(self): diff --git a/ckanext/geodatagov/tests/test_logic.py b/ckanext/geodatagov/tests/test_logic.py index c952a505..c51d633e 100644 --- a/ckanext/geodatagov/tests/test_logic.py +++ b/ckanext/geodatagov/tests/test_logic.py @@ -1,37 +1,16 @@ import json -import os +from utils import populate_locations_table from ckan.tests.helpers import FunctionalTestBase -import ckan.lib.search as search from ckan.tests import factories -from ckan.tests.helpers import reset_db -from ckan.model.meta import Session, metadata from ckanext.geodatagov.logic import rollup_save_action class TestLogic(FunctionalTestBase): - @classmethod - def setup(cls): - search.clear_all() - def setup_method(self): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - # echo "Downloading locations table" - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'DROP TABLE IF EXISTS locations;'") - os.system("wget https://github.com/GSA/datagov-deploy/raw/71936f004be1882a506362670b82c710c64ef796/" - "ansible/roles/software/ec2/ansible/files/locations.sql.gz -O /tmp/locations.sql.gz") - # echo "Creating locations table" - os.system("gunzip -c /tmp/locations.sql.gz | PGPASSWORD=ckan psql -h db -U ckan -d ckan -v ON_ERROR_STOP=1") - # echo "Cleaning" - os.system("rm -f /tmp/locations.sql.gz") - # os.system("ckan -c test.ini db upgrade -p harvest") - metadata.create_all(bind=Session.bind) + populate_locations_table() def create_datasets(self): self.group1 = factories.Group() diff --git a/ckanext/geodatagov/tests/test_update_geo.py b/ckanext/geodatagov/tests/test_update_geo.py index 5265e36d..dc431b0e 100644 --- a/ckanext/geodatagov/tests/test_update_geo.py +++ b/ckanext/geodatagov/tests/test_update_geo.py @@ -1,7 +1,6 @@ import logging -from ckan.tests.helpers import reset_db - +from utils import populate_locations_table from ckanext.geodatagov.logic import translate_spatial @@ -10,9 +9,8 @@ class TestUpdateGeo(object): - @classmethod - def setup(cls): - reset_db() + def setup_method(self): + populate_locations_table() def test_translations(self): """ test translate_spatial function """ diff --git a/ckanext/geodatagov/tests/test_waf-collection.py b/ckanext/geodatagov/tests/test_waf-collection.py index 85fb9255..f9aa7ffd 100644 --- a/ckanext/geodatagov/tests/test_waf-collection.py +++ b/ckanext/geodatagov/tests/test_waf-collection.py @@ -1,38 +1,23 @@ import json import logging import pytest -import os - -from ckan.model.meta import Session, metadata from factories import HarvestJobObj, WafCollectionHarvestSourceObj -import mock_static_file_server from ckan import model from ckanext.geodatagov.harvesters.waf_collection import WAFCollectionHarvester from ckanext.spatial.validation import all_validators import ckanext.harvest.model as harvest_model - -from ckan.tests.helpers import reset_db, call_action from ckan.tests.factories import Organization +from utils import PORT + log = logging.getLogger(__name__) @pytest.mark.usefixtures("with_plugins") class TestWafCollectionHarvester(object): - @classmethod - def setup_class(cls): - log.info('Starting mock http server') - mock_static_file_server.serve() def setup_method(self): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - # os.system("ckan -c test.ini db upgrade -p harvest") - metadata.create_all(bind=Session.bind) self.organization = Organization() @@ -104,10 +89,9 @@ def run_import(self): def get_datasets_from_waf_collection1_sample(self): """ harvest waf-collection1/ folder as waf-collection source """ - url = 'http://127.0.0.1:%s/waf-collection1/index.html' % mock_static_file_server.PORT + url = f'http://127.0.0.1:{PORT}/waf-collection1/index.html' - collection_metadata = "http://127.0.0.1:%s/waf-collection1/cfg/SeriesCollection_tl_2013_county.shp.iso.xml" %\ - mock_static_file_server.PORT + collection_metadata = f"http://127.0.0.1:{PORT}/waf-collection1/cfg/SeriesCollection_tl_2013_county.shp.iso.xml" config = '{"collection_metadata_url": "%s", "validator_profiles": ["iso19139ngdc"], "private_datasets": false}' %\ collection_metadata self.run_gather(url=url, source_config=config) diff --git a/ckanext/geodatagov/tests/test_waf.py b/ckanext/geodatagov/tests/test_waf.py index 7be8e7ba..d93bb403 100644 --- a/ckanext/geodatagov/tests/test_waf.py +++ b/ckanext/geodatagov/tests/test_waf.py @@ -1,37 +1,28 @@ import json import logging +import pytest +import os import ckanext.harvest.model as harvest_model -import mock_static_file_server +from ckan.tests.helpers import reset_db from ckan import model from ckan.logic import get_action from ckanext.geodatagov.harvesters.base import GeoDataGovWAFHarvester from factories import HarvestJobObj, WafHarvestSourceObj - -from ckan.tests.helpers import reset_db from ckan.tests.factories import Organization, SysadminWithToken -import pytest -import os +import ckan.lib.search as search from ckan.model.meta import Session, metadata + +from utils import PORT, reset_db_and_solr + log = logging.getLogger(__name__) @pytest.mark.usefixtures("with_plugins") class TestWafHarvester(object): - @classmethod - def setup_class(cls): - log.info('Starting mock http server') - mock_static_file_server.serve() - def setup_method(self): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - # os.system("ckan -c test.ini db upgrade -p harvest") - metadata.create_all(bind=Session.bind) + reset_db_and_solr() self.organization = Organization() @@ -102,23 +93,27 @@ def run_import(self): def get_datasets_from_waf1_sample(self): """ harvest waf1/ folder as waf source """ - url = 'http://127.0.0.1:%s/waf1/index.html' % mock_static_file_server.PORT + url = f'http://127.0.0.1:{PORT}/waf1/index.html' self.config1 = '{"validator_profiles": ["iso19139ngdc"], "private_datasets": "false"}' self.run_gather(url=url, source_config=self.config1) self.run_fetch() datasets = self.run_import() + self.job.status = 'Finished' + self.job.save() return datasets def get_datasets_from_waf_trim_tags(self): """ harvest waf-trim-tags/ folder as waf source """ - url = 'http://127.0.0.1:%s/waf-trim-tags/index.html' % mock_static_file_server.PORT + url = f'http://127.0.0.1:{PORT}/waf-trim-tags/index.html' self.config1 = '{"validator_profiles": ["iso19139ngdc"], "private_datasets": "false"}' self.run_gather(url=url, source_config=self.config1) self.run_fetch() datasets = self.run_import() + self.job.status = 'Finished' + self.job.save() return datasets @@ -131,12 +126,14 @@ def test_waf1_datasets_count(self): def test_datasets_from_waf_fgdc_sample(self): """ harvest waf-fgdc/ folder as waf source """ - url = 'http://127.0.0.1:%s/waf-fgdc/index.html' % mock_static_file_server.PORT + url = f'http://127.0.0.1:{PORT}/waf-fgdc/index.html' self.config1 = '{"private_datasets": "false"}' self.run_gather(url=url, source_config=self.config1) self.run_fetch() datasets = self.run_import() + self.job.status = 'Finished' + self.job.save() assert len(datasets) == 1 @@ -204,7 +201,6 @@ def test_waf_trim_tags(self): expected_list = ['tag1 /tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6'] bad_list = list(set(tag_list) - set(expected_list)) log.info("Tags that are not trimmed: %s", bad_list) - assert (sorted(tag_list) == sorted(expected_list)) def test_extras_rollup(self): diff --git a/ckanext/geodatagov/tests/test_waf_GMI.py b/ckanext/geodatagov/tests/test_waf_GMI.py index 525b2a9e..c2703dfd 100644 --- a/ckanext/geodatagov/tests/test_waf_GMI.py +++ b/ckanext/geodatagov/tests/test_waf_GMI.py @@ -1,36 +1,22 @@ import json import logging +import pytest import ckanext.harvest.model as harvest_model -import mock_static_file_server from ckan import model from ckanext.geodatagov.harvesters.base import GeoDataGovWAFHarvester from factories import HarvestJobObj, WafHarvestSourceObj - -from ckan.tests.helpers import reset_db from ckan.tests.factories import Organization -import pytest -import os -from ckan.model.meta import Session, metadata + +from utils import PORT, reset_db_and_solr log = logging.getLogger(__name__) @pytest.mark.usefixtures("with_plugins") class TestWafHarvester(object): - @classmethod - def setup_class(cls): - log.info('Starting mock http server') - mock_static_file_server.serve() - def setup_method(self): - # https://github.com/ckan/ckan/issues/4764 - # drop extension postgis so we can reset db - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") - reset_db() - os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") - # os.system("ckan -c test.ini db upgrade -p harvest") - metadata.create_all(bind=Session.bind) + reset_db_and_solr() self.organization = Organization() @@ -100,7 +86,7 @@ def run_import(self): def get_datasets_from_waf_gmi_sample(self): """ harvest waf-gmi/ folder as waf source """ - url = 'http://127.0.0.1:%s/waf-gmi/index.html' % mock_static_file_server.PORT + url = f'http://127.0.0.1:{PORT}/waf-gmi/index.html' self.config1 = '{"private_datasets": "false"}' self.run_gather(url=url, source_config=self.config1) diff --git a/ckanext/geodatagov/tests/utils.py b/ckanext/geodatagov/tests/utils.py new file mode 100644 index 00000000..470a55c8 --- /dev/null +++ b/ckanext/geodatagov/tests/utils.py @@ -0,0 +1,73 @@ +import http.server +import logging +import socketserver +from threading import Thread +import os + +from ckan.tests.helpers import reset_db +from ckan.model.meta import Session, metadata +import ckan.lib.search as search + + +log = logging.getLogger(__name__) + +PORT = 8999 + +def simple_http_server(port=PORT): + '''Serves test XML files over HTTP''' + + # Make sure we serve from the tests' XML directory + os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'data-samples')) + + Handler = http.server.SimpleHTTPRequestHandler + + class TestServer(socketserver.TCPServer): + allow_reuse_address = True + + skip_connection = False + try: + httpd = TestServer(("", port), Handler) + except Exception as e: + print('Serve error {}'.format(e)) + skip_connection = True + + if skip_connection is False: + info = 'Serving test HTTP server at port', port + print(info) + log.info(info) + + httpd_thread = Thread(target=httpd.serve_forever) + httpd_thread.setDaemon(True) + httpd_thread.start() + +def populate_locations_table(): + os.system("wget https://github.com/GSA/datagov-deploy/raw/71936f004be1882a506362670b82c710c64ef796/" + "ansible/roles/software/ec2/ansible/files/locations.sql.gz -O /tmp/locations.sql.gz") + # echo "Creating locations table" + os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'DROP TABLE IF EXISTS locations;'") + os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'DROP SEQUENCE IF EXISTS locations_id_seq;'") + os.system("gunzip -c /tmp/locations.sql.gz | PGPASSWORD=ckan psql -h db -U ckan -d ckan -v ON_ERROR_STOP=1") + # echo "Cleaning" + os.system("rm -f /tmp/locations.sql.gz") + +def reset_db_and_solr(): + # https://github.com/ckan/ckan/issues/4764 + # drop extension postgis so we can reset db + try: + os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid();'") + except Exception: + pass + os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'drop extension IF EXISTS postgis cascade;'") + try: + reset_db() + except Exception: + pass + os.system("PGPASSWORD=ckan psql -h db -U ckan -d ckan -c 'create extension postgis;'") + # os.system("ckan -c test.ini db upgrade -p harvest") + metadata.create_all(bind=Session.bind) + + search.clear_all() + + + diff --git a/test.sh b/test.sh index 9437ffe8..91cf47fe 100755 --- a/test.sh +++ b/test.sh @@ -29,8 +29,9 @@ PASS=ckan ckan -c $test_ini db upgrade -p harvest -# pytest --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/ -pytest -s --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_waf-collection.py + +pytest --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/ +# pytest -s --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_waf-collection.py # Run this this pytest command if only testing a single test # pytest --ckan-ini=$test_ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_category_tags.py