fix tests

GSA · Mar 4, 2025 · 67e3175 · 67e3175
1 parent e49a153
commit 67e3175
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 53 deletions.
diff --git a/ckanext/geodatagov/harvesters/waf_collection.py b/ckanext/geodatagov/harvesters/waf_collection.py
@@ -104,16 +104,26 @@ def gather_stage(self, harvest_job):
             .first()
         )
 
+        if existing_harvest_object:
+            status = "change"
+            guid = existing_harvest_object.guid
+            package_id = existing_harvest_object.package_id
+        else:
+            status, package_id = "new", None
+
         obj = HarvestObject(
-            guid=guid,
-            job=harvest_job
-        )
-        obj.extras = [
+            job=harvest_job,
+            extras=[
                 HOExtra(key="collection_metadata", value="true"),
                 HOExtra(key="waf_location", value=collection_metadata_url),
-        ]
+                HOExtra(key="status", value=status),
+            ],
+            guid=guid,
+            package_id=package_id,
+        )
 
         queue.fetch_and_import_stages(self, obj)
+
         if obj.state == "ERROR":
             self._save_gather_error(
                 "Collection object failed to harvest, not harvesting", harvest_job

diff --git a/ckanext/geodatagov/tests/test_fix_spatial.py b/ckanext/geodatagov/tests/test_fix_spatial.py
@@ -1,10 +1,9 @@
 import os
+import pytest
 
 import ckan.plugins as p
 import ckan.tests.factories as factories
 import ckan.tests.helpers as helpers
-from ckan.model.meta import Session, metadata
-import pytest
 
 @pytest.mark.usefixtures("with_plugins")
 class TestSpatialField(object):

diff --git a/ckanext/geodatagov/tests/test_waf-collection.py b/ckanext/geodatagov/tests/test_waf-collection.py
@@ -8,16 +8,17 @@
 from ckanext.spatial.validation import all_validators
 import ckanext.harvest.model as harvest_model
 from ckan.tests.factories import Organization
+from ckan.tests.helpers import call_action
 
-from utils import PORT
+from utils import PORT, reset_db_and_solr
 
 log = logging.getLogger(__name__)
 
 @pytest.mark.usefixtures("with_plugins")
 class TestWafCollectionHarvester(object):
 
-
     def setup_method(self):
+        reset_db_and_solr()
 
         self.organization = Organization()
 
@@ -31,16 +32,16 @@ def run_gather(self, url, source_config):
                                                owner_org=self.organization['id'],
                                                # config=source_config,
                                                **sc)
-        job = HarvestJobObj(source=source)
+        self.job = HarvestJobObj(source=source)
 
         self.harvester = WAFCollectionHarvester()
 
         # gather stage
         log.info('GATHERING %s', url)
-        obj_ids = self.harvester.gather_stage(job)
-        log.info('job.gather_errors=%s', job.gather_errors)
-        if len(job.gather_errors) > 0:
-            raise Exception(job.gather_errors[0])
+        obj_ids = self.harvester.gather_stage(self.job)
+        log.info('job.gather_errors=%s', self.job.gather_errors)
+        if len(self.job.gather_errors) > 0:
+            raise Exception(self.job.gather_errors[0])
 
         log.info('obj_ids=%s', obj_ids)
         if obj_ids is None or len(obj_ids) == 0:
@@ -97,6 +98,8 @@ def get_datasets_from_waf_collection1_sample(self):
         self.run_gather(url=url, source_config=config)
         self.run_fetch()
         datasets = self.run_import()
+        self.job.status = 'Finished'
+        self.job.save()
 
         return datasets
 
@@ -123,46 +126,45 @@ def test_waf_collection1_datasets_as_child(self):
         assert 'collection_package_id' in keys
         assert 'collection_metadata' not in keys
 
-    # def test_waf_collection1_parent_exists(self):
-    #     """ Harvest waf-collection1/ folder as waf-collection source
-    #         and test parent dataset exists (include the collection_metadata=true extra) """
+    def test_waf_collection1_parent_exists(self):
+        """ Harvest waf-collection1/ folder as waf-collection source
+            and test parent dataset exists (include the collection_metadata=true extra) """
 
-    #     datasets = self.get_datasets_from_waf_collection1_sample()
-    #     dataset = datasets[0]
-    #     extras = json.loads(dataset.extras['extras_rollup'])
+        datasets = self.get_datasets_from_waf_collection1_sample()
+        dataset = datasets[0]
+        extras = json.loads(dataset.extras['extras_rollup'])
 
-    #     parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id'])
-    #     parent_keys = [extra['key'] for extra in parent['extras']]
-    #     assert 'collection_metadata' in parent_keys
-    #     assert 'true' == [extra['value'] for extra in parent['extras'] if extra['key'] == 'collection_metadata'][0]
+        parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id'])
+        parent_keys = [extra['key'] for extra in parent['extras']]
+        assert 'collection_metadata' in parent_keys
+        assert 'true' == [extra['value'] for extra in parent['extras'] if extra['key'] == 'collection_metadata'][0]
 
-    # def test_waf_collection1_parent_title(self):
-    #     """ Harvest waf-collection1/ folder as waf-collection source
-    #         and test parent dataset have the expected title and name """
+    def test_waf_collection1_parent_title(self):
+        """ Harvest waf-collection1/ folder as waf-collection source
+            and test parent dataset have the expected title and name """
 
-    #     datasets = self.get_datasets_from_waf_collection1_sample()
-    #     dataset = datasets[0]
-    #     extras = json.loads(dataset.extras['extras_rollup'])
+        datasets = self.get_datasets_from_waf_collection1_sample()
+        dataset = datasets[0]
+        extras = json.loads(dataset.extras['extras_rollup'])
 
-    #     parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id'])
+        parent = call_action('package_show', context={'user': 'dummy'}, id=extras['collection_package_id'])
 
-    #     assert parent['title'] == ('TIGER/Line Shapefile, 2013, '
-    #                                'Series Information File for the Current county and Equivalent National Shapefile')
-    #     assert parent['name'] == ('tiger-line-shapefile-2013-'
-    #                               'series-information-file-for-the-current-county-and-equivalent-nationa')
+        assert parent['title'] == ('TIGER/Line Shapefile, 2013, '
+                                   'Series Information File for the Current county and Equivalent National Shapefile')
+        assert parent['name'] == ('tiger-line-shapefile-2013-'
+                                  'series-information-file-for-the-current-county-and-equivalent-nationa')
 
-    # def test_waf_collection_transformation_failed(self):
-    #     url = 'http://127.0.0.1:%s/waf-collection2/index.html' % mock_static_file_server.PORT
+    def test_waf_collection_transformation_failed(self):
+        url = f'http://127.0.0.1:{PORT}/waf-collection2/index.html'
 
-    #     collection_metadata = "http://127.0.0.1:%s/waf-collection2/cfg/SeriesCollection_tl_2013_county.shp.iso.xml" %\
-    #         mock_static_file_server.PORT
-    #     config = '{"collection_metadata_url": "%s", "validator_profiles": ["iso19139ngdc"], "private_datasets": false}' %\
-    #         collection_metadata
-    #     self.run_gather(url=url, source_config=config)
+        collection_metadata = f"http://127.0.0.1:{PORT}/waf-collection2/cfg/SeriesCollection_tl_2013_county.shp.iso.xml"
+        config = '{"collection_metadata_url": "%s", "validator_profiles": ["iso19139ngdc"], "private_datasets": false}' %\
+            collection_metadata
+        self.run_gather(url=url, source_config=config)
 
-    #     self.run_fetch()
+        self.run_fetch()
 
-    #     # we don't manage IS0 19110
-    #     with pytest.raises(Exception) as e:
-    #         self.run_import()
-    #     assert 'Transformation to ISO failed' in str(e.value)
+        # we don't manage IS0 19110
+        with pytest.raises(Exception) as e:
+            self.run_import()
+        assert 'Transformation to ISO failed' in str(e.value)
diff --git a/ckanext/geodatagov/tests/test_waf.py b/ckanext/geodatagov/tests/test_waf.py
@@ -1,18 +1,13 @@
 import json
 import logging
 import pytest
-import os
 
 import ckanext.harvest.model as harvest_model
-from ckan.tests.helpers import reset_db
 from ckan import model
 from ckan.logic import get_action
 from ckanext.geodatagov.harvesters.base import GeoDataGovWAFHarvester
 from factories import HarvestJobObj, WafHarvestSourceObj
 from ckan.tests.factories import Organization, SysadminWithToken
-import ckan.lib.search as search
-from ckan.model.meta import Session, metadata
-
 
 from utils import PORT, reset_db_and_solr
 

diff --git a/test.sh b/test.sh
@@ -31,7 +31,6 @@ ckan -c $test_ini db upgrade -p harvest
 
 
 pytest --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/
-# pytest -s --ckan-ini=test.ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_waf-collection.py
 
 # Run this this pytest command if only testing a single test
 # pytest --ckan-ini=$test_ini --cov=ckanext.geodatagov --disable-warnings ckanext/geodatagov/tests/test_category_tags.py