diff --git a/ckanext/geodatagov/harvesters/waf_collection.py b/ckanext/geodatagov/harvesters/waf_collection.py index 0e8a9c0a..07fccec6 100644 --- a/ckanext/geodatagov/harvesters/waf_collection.py +++ b/ckanext/geodatagov/harvesters/waf_collection.py @@ -104,16 +104,26 @@ def gather_stage(self, harvest_job): .first() ) + if existing_harvest_object: + status = "change" + guid = existing_harvest_object.guid + package_id = existing_harvest_object.package_id + else: + status, package_id = "new", None + obj = HarvestObject( - guid=guid, - job=harvest_job - ) - obj.extras = [ + job=harvest_job, + extras=[ HOExtra(key="collection_metadata", value="true"), HOExtra(key="waf_location", value=collection_metadata_url), - ] + HOExtra(key="status", value=status), + ], + guid=guid, + package_id=package_id, + ) queue.fetch_and_import_stages(self, obj) + if obj.state == "ERROR": self._save_gather_error( "Collection object failed to harvest, not harvesting", harvest_job diff --git a/ckanext/geodatagov/tests/test_fix_spatial.py b/ckanext/geodatagov/tests/test_fix_spatial.py index a9ebaec9..0568e8c1 100644 --- a/ckanext/geodatagov/tests/test_fix_spatial.py +++ b/ckanext/geodatagov/tests/test_fix_spatial.py @@ -1,10 +1,9 @@ import os +import pytest import ckan.plugins as p import ckan.tests.factories as factories import ckan.tests.helpers as helpers -from ckan.model.meta import Session, metadata -import pytest @pytest.mark.usefixtures("with_plugins") class TestSpatialField(object): diff --git a/ckanext/geodatagov/tests/test_waf-collection.py b/ckanext/geodatagov/tests/test_waf-collection.py index f9aa7ffd..064debf4 100644 --- a/ckanext/geodatagov/tests/test_waf-collection.py +++ b/ckanext/geodatagov/tests/test_waf-collection.py @@ -8,16 +8,17 @@ from ckanext.spatial.validation import all_validators import ckanext.harvest.model as harvest_model from ckan.tests.factories import Organization +from ckan.tests.helpers import call_action -from utils import PORT +from utils import PORT, reset_db_and_solr log = logging.getLogger(__name__) @pytest.mark.usefixtures("with_plugins") class TestWafCollectionHarvester(object): - def setup_method(self): + reset_db_and_solr() self.organization = Organization() @@ -31,16 +32,16 @@ def run_gather(self, url, source_config): owner_org=self.organization['id'], # config=source_config, **sc) - job = HarvestJobObj(source=source) + self.job = HarvestJobObj(source=source) self.harvester = WAFCollectionHarvester() # gather stage log.info('GATHERING %s', url) - obj_ids = self.harvester.gather_stage(job) - log.info('job.gather_errors=%s', job.gather_errors) - if len(job.gather_errors) > 0: - raise Exception(job.gather_errors[0]) + obj_ids = self.harvester.gather_stage(self.job) + log.info('job.gather_errors=%s', self.job.gather_errors) + if len(self.job.gather_errors) > 0: + raise Exception(self.job.gather_errors[0]) log.info('obj_ids=%s', obj_ids) if obj_ids is None or len(obj_ids) == 0: @@ -97,6 +98,8 @@ def get_datasets_from_waf_collection1_sample(self): self.run_gather(url=url, source_config=config) self.run_fetch() datasets = self.run_import() + self.job.status = 'Finished' + self.job.save() return datasets @@ -123,46 +126,45 @@ def test_waf_collection1_datasets_as_child(self): assert 'collection_package_id' in keys assert 'collection_metadata' not in keys - # def test_waf_collection1_parent_exists(self): - # """ Harvest waf-collection1/ folder as waf-collection source - # and test parent dataset exists (include the collection_metadata=true extra) """ + def test_waf_collection1_parent_exists(self): + """ Harvest waf-collection1/ folder as waf-collection source + and test parent dataset exists (include the collection_metadata=true extra) """ - # datasets = self.get_datasets_from_waf_collection1_sample() - # dataset = datasets[0] - # extras = json.loads(dataset.extras['extras_rollup']) + datasets = self.get_datasets_from_waf_collection1_sample() + dataset = datasets[0] + extras = json.loads(dataset.extras['extras_rollup']) - # parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id']) - # parent_keys = [extra['key'] for extra in parent['extras']] - # assert 'collection_metadata' in parent_keys - # assert 'true' == [extra['value'] for extra in parent['extras'] if extra['key'] == 'collection_metadata'][0] + parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id']) + parent_keys = [extra['key'] for extra in parent['extras']] + assert 'collection_metadata' in parent_keys + assert 'true' == [extra['value'] for extra in parent['extras'] if extra['key'] == 'collection_metadata'][0] - # def test_waf_collection1_parent_title(self): - # """ Harvest waf-collection1/ folder as waf-collection source - # and test parent dataset have the expected title and name """ + def test_waf_collection1_parent_title(self): + """ Harvest waf-collection1/ folder as waf-collection source + and test parent dataset have the expected title and name """ - # datasets = self.get_datasets_from_waf_collection1_sample() - # dataset = datasets[0] - # extras = json.loads(dataset.extras['extras_rollup']) + datasets = self.get_datasets_from_waf_collection1_sample() + dataset = datasets[0] + extras = json.loads(dataset.extras['extras_rollup']) - # parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id']) + parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id']) - # assert parent['title'] == ('TIGER/Line Shapefile, 2013, ' - # 'Series Information File for the Current county and Equivalent National Shapefile') - # assert parent['name'] == ('tiger-line-shapefile-2013-' - # 'series-information-file-for-the-current-county-and-equivalent-nationa') + assert parent['title'] == ('TIGER/Line Shapefile, 2013, ' + 'Series Information File for the Current county and Equivalent National Shapefile') + assert parent['name'] == ('tiger-line-shapefile-2013-' + 'series-information-file-for-the-current-county-and-equivalent-nationa') - # def test_waf_collection_transformation_failed(self): - # url = 'http://127.0.0.1:%s/waf-collection2/index.html' % mock_static_file_server.PORT + def test_waf_collection_transformation_failed(self): + url = f'http://127.0.0.1:{PORT}/waf-collection2/index.html' - # collection_metadata = "http://127.0.0.1:%s/waf-collection2/cfg/SeriesCollection_tl_2013_county.shp.iso.xml" %\ - # mock_static_file_server.PORT - # config = '{"collection_metadata_url": "%s", "validator_profiles": ["iso19139ngdc"], "private_datasets": false}' %\ - # collection_metadata - # self.run_gather(url=url, source_config=config) + collection_metadata = f"http://127.0.0.1:{PORT}/waf-collection2/cfg/SeriesCollection_tl_2013_county.shp.iso.xml" + config = '{"collection_metadata_url": "%s", "validator_profiles": ["iso19139ngdc"], "private_datasets": false}' %\ + collection_metadata + self.run_gather(url=url, source_config=config) - # self.run_fetch() + self.run_fetch() - # # we don't manage IS0 19110 - # with pytest.raises(Exception) as e: - # self.run_import() - # assert 'Transformation to ISO failed' in str(e.value) + # we don't manage IS0 19110 + with pytest.raises(Exception) as e: + self.run_import() + assert 'Transformation to ISO failed' in str(e.value) diff --git a/ckanext/geodatagov/tests/test_waf.py b/ckanext/geodatagov/tests/test_waf.py index d93bb403..6443b4a3 100644 --- a/ckanext/geodatagov/tests/test_waf.py +++ b/ckanext/geodatagov/tests/test_waf.py @@ -1,18 +1,13 @@ import json import logging import pytest -import os import ckanext.harvest.model as harvest_model -from ckan.tests.helpers import reset_db from ckan import model from ckan.logic import get_action from ckanext.geodatagov.harvesters.base import GeoDataGovWAFHarvester from factories import HarvestJobObj, WafHarvestSourceObj from ckan.tests.factories import Organization, SysadminWithToken -import ckan.lib.search as search -from ckan.model.meta import Session, metadata - from utils import PORT, reset_db_and_solr diff --git a/test.sh b/test.sh index 91cf47fe..96126a86 100755 --- a/test.sh +++ b/test.sh @@ -31,7 +31,6 @@ ckan -c $test_ini db upgrade -p harvest pytest --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/ -# pytest -s --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_waf-collection.py # Run this this pytest command if only testing a single test # pytest --ckan-ini=$test_ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_category_tags.py