Dashboard to see how many urls from each environment

environmental_justice/models.py

@@ -25,6 +25,8 @@ class EnvironmentalJusticeRow(models.Model):
     temporal_extent = models.CharField("Temporal Extent")
     temporal_resolution = models.CharField("Temporal Resolution")
 
+    sde_links = models.CharField("SDE Links")
+
     class Meta:
         verbose_name = "Environmental Justice Row"
         verbose_name_plural = "Environmental Justice Rows"

sde_collections/migrations/0040_collection_url_count_prod_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.6 on 2023-12-01 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sde_collections", "0039_includepattern"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="collection",
+            name="url_count_prod",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="collection",
+            name="url_count_secret_prod",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="collection",
+            name="url_count_secret_test",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="collection",
+            name="url_count_test",
+            field=models.IntegerField(default=0),
+        ),
+    ]

sde_collections/migrations/0041_rename_url_count_prod_collection_url_count_production_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.6 on 2023-12-01 17:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sde_collections", "0040_collection_url_count_prod_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="collection",
+            old_name="url_count_prod",
+            new_name="url_count_production",
+        ),
+        migrations.RenameField(
+            model_name="collection",
+            old_name="url_count_secret_prod",
+            new_name="url_count_secret_production",
+        ),
+    ]

sde_collections/migrations/0042_merge_20240219_1422.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.0.1 on 2024-02-19 20:22
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0041_alter_candidateurl_hash"),
+        (
+            "sde_collections",
+            "0041_rename_url_count_prod_collection_url_count_production_and_more",
+        ),
+    ]
+
+    operations = []

sde_collections/migrations/0043_rename_url_count_secret_production_collection_url_count_dev_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.1 on 2024-02-19 20:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0042_merge_20240219_1422"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="collection",
+            old_name="url_count_secret_production",
+            new_name="url_count_dev",
+        ),
+        migrations.RemoveField(
+            model_name="collection",
+            name="url_count_secret_test",
+        ),
+    ]

sde_collections/models/collection.py

@@ -8,6 +8,7 @@
 
 from config_generation.db_to_xml import XmlEditor
 
+from ..sinequa_api import server_configs
 from ..utils.github_helper import GitHubHandler
 from .collection_choice_fields import (
     ConnectorChoices,
@@ -19,6 +20,9 @@
     WorkflowStatusChoices,
 )
 
+# from ..tasks import import_candidate_urls_counts_from_api
+
+
 User = get_user_model()
 
 
@@ -94,6 +98,10 @@ class Collection(models.Model):
     )
     curation_started = models.DateTimeField("Curation Started", null=True, blank=True)
 
+    url_count_dev = models.IntegerField(default=0)
+    url_count_test = models.IntegerField(default=0)
+    url_count_production = models.IntegerField(default=0)
+
     class Meta:
         """Meta definition for Collection."""
 
@@ -154,6 +162,101 @@ def workflow_status_button_color(self) -> str:
         }
         return color_choices[self.workflow_status]
 
+    def get_server_url(self, server_name) -> str:
+        base_url = server_configs[server_name]["base_url"]
+        app_name = server_configs[server_name]["app_name"]
+        query_name = server_configs[server_name]["query_name"]
+
+        payload = {
+            "name": query_name,
+            "scope": "All",
+            "text": "",
+            "advanced": {
+                "collection": f"/SMD/{self.config_folder}/",
+            },
+        }
+        encoded_payload = urllib.parse.quote(json.dumps(payload))
+        return f"{base_url}/app/{app_name}/#/search?query={encoded_payload}"
+
+    # def fetch_url_counts(self):
+    #     for collection in Collection.objects.all():
+    #         if not (
+    #             collection.url_count_test == 0
+    #             and collection.url_count_secret_test == 0
+    #             and collection.url_count_production == 0
+    #             and collection.url_count_secret_production == 0
+    #         ):
+    #             continue
+    #         collection_ids = [collection.id]
+    #         server_names = [
+    #             # "dev",
+    #             "test",
+    #             "production",
+    #         ]
+    #         for server_name in server_names:
+    #             count = import_candidate_urls_counts_from_api(server_name, collection_ids)
+    #             setattr(collection, f"url_count_{server_name}", count)
+    #             collection.save()
+
+    # def fetch_url_counts(self):
+    #     """Fetch the URL counts from the production webapp."""
+    #     ENVIRONMENTS = {
+    #         "test": {
+    #             "url": "https://sciencediscoveryengine.test.nasa.gov",
+    #             "query": "query-smd-primary",
+    #             "app": "nasa-sba-smd",
+    #             "variable": "url_count_test",
+    #             "folder": "SMD",
+    #         },
+    #         "secret_test": {
+    #             "url": "https://sciencediscoveryengine.test.nasa.gov",
+    #             "query": "query-sde-primary",
+    #             "app": "nasa-sba-sde",
+    #             "variable": "url_count_secret_test",
+    #             "folder": "SDE",
+    #         },
+    #         "prod": {
+    #             "url": "https://sciencediscoveryengine.nasa.gov",
+    #             "query": "query-smd-primary",
+    #             "app": "nasa-sba-smd",
+    #             "variable": "url_count_prod",
+    #             "folder": "SMD",
+    #         },
+    #         "secret_prod": {
+    #             "url": "https://sciencediscoveryengine.nasa.gov",
+    #             "query": "query-sde-primary",
+    #             "app": "nasa-sba-sde",
+    #             "variable": "url_count_secret_prod",
+    #             "folder": "SDE",
+    #         },
+    #     }
+
+    #     totals = []
+
+    #     for environment_name, environment_config in ENVIRONMENTS.items():
+    #         count = import_candidate_urls_counts_from_api(
+    #             server_name=environment_name, collection_ids=[self.id]
+    #         )
+
+    #         # setattr(self, environment_config["variable"], response_json["total"])
+    #         totals.append(count)
+    #         # self.save()
+    #     return totals
+
+    # from sde_collections.tasks import import_candidate_urls_counts_from_api
+    # for collection in Collection.objects.all():
+    # collection_ids = [collection.id]
+    # server_names = [
+    #     "test",
+    #     "secret_test",
+    #     "production",
+    #     "secret_production",
+    # ]
+    # for server_name in server_names:
+    #     count = import_candidate_urls_counts_from_api(server_name, collection_ids)
+    #     setattr(collection, f"url_count_{server_name}", count)
+    #     collection.save()
+
     def _process_exclude_list(self):
         """Process the exclude list."""
         return [
@@ -423,3 +526,30 @@ class RequiredUrls(models.Model):
 
     def __str__(self) -> str:
         return self.url
+
+
+class Server(models.Model):
+    name = models.CharField(max_length=255)
+    base_url = models.CharField(max_length=255)
+    app_name = models.CharField(max_length=255)
+    query_name = models.CharField(max_length=255)
+    username = models.CharField(max_length=255, blank=True, default="")
+    password = models.CharField(max_length=255, blank=True, default="")
+
+    def __str__(self) -> str:
+        return self.name
+
+
+class IndexingMetadata(models.Model):
+    server = models.ForeignKey("Server", on_delete=models.CASCADE)
+    collection = models.ForeignKey("Collection", on_delete=models.CASCADE)
+    last_indexed = models.DateTimeField(null=True, blank=True)
+    last_indexed_count = models.IntegerField(null=True, blank=True)
+
+    def __str__(self) -> str:
+        return f"{self.collection.name} - {self.server.name}"
+
+    def update_last_indexed(self, last_indexed, last_indexed_count):
+        self.last_indexed = last_indexed
+        self.last_indexed_count = last_indexed_count
+        self.save()

sde_collections/sinequa_api.py

@@ -9,32 +9,22 @@
     "dev": {
         "app_name": "nasa-sba-smd",
         "query_name": "query-smd-primary",
-        "base_url": "http://sde-renaissance.nasa-impact.net",
+        "base_url": "http://sde-xli.nasa-impact.net",
     },
-    "test": {
+    "lis_server": {
         "app_name": "nasa-sba-smd",
         "query_name": "query-smd-primary",
-        "base_url": "https://sciencediscoveryengine.test.nasa.gov",
+        "base_url": "http://sde-xli.nasa-impact.net",
     },
-    "production": {
+    "test": {
         "app_name": "nasa-sba-smd",
         "query_name": "query-smd-primary",
-        "base_url": "https://sciencediscoveryengine.nasa.gov",
-    },
-    "secret_test": {
-        "app_name": "nasa-sba-sde",
-        "query_name": "query-sde-primary",
         "base_url": "https://sciencediscoveryengine.test.nasa.gov",
     },
-    "secret_production": {
-        "app_name": "nasa-sba-sde",
-        "query_name": "query-sde-primary",
-        "base_url": "https://sciencediscoveryengine.nasa.gov",
-    },
-    "lis_server": {
+    "production": {
         "app_name": "nasa-sba-smd",
         "query_name": "query-smd-primary",
-        "base_url": "http://sde-xli.nasa-impact.net",
+        "base_url": "https://sciencediscoveryengine.nasa.gov",
     },
 }
 
@@ -45,6 +35,7 @@ def __init__(self, server_name: str) -> None:
         self.app_name: str = server_configs[server_name]["app_name"]
         self.query_name: str = server_configs[server_name]["query_name"]
         self.base_url: str = server_configs[server_name]["base_url"]
+        self.folder: str = "SDE" if self.app_name == "nasa-sba-sde" else "SMD"
 
     def process_response(self, url: str, payload: dict[str, Any]) -> Any:
         response = requests.post(url, headers={}, json=payload, verify=False)

sde_collections/tasks.py

@@ -13,6 +13,10 @@
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
+# from django.apps import apps
+
+# Collection = apps.get_model("Collection")
+
 
 def _get_data_to_import(collection, server_name):
     # ignore these because they are API collections and don't have URLs
@@ -24,6 +28,13 @@ def _get_data_to_import(collection, server_name):
         "/SMD/CASEI_Platform/",
         "/SMD/CMR_API/",
         "/SMD/PDS_API_Legacy_All/",
+        "/SDE/ASTRO_NAVO_HEASARC/",
+        "/SDE/CASEI_Campaign/",
+        "/SDE/CASEI_Deployment/",
+        "/SDE/CASEI_Instrument/",
+        "/SDE/CASEI_Platform/",
+        "/SDE/CMR_API/",
+        "/SDE/PDS_API_Legacy_All/",
     ]
 
     data_to_import = []
@@ -97,6 +108,38 @@ def import_candidate_urls_from_api(server_name="test", collection_ids=[]):
     shutil.rmtree(TEMP_FOLDER_NAME)
 
 
+@celery_app.task(soft_time_limit=10000)
+def import_candidate_urls_counts_from_api(server_name, collection_ids=[]):
+    collections = Collection.objects.filter(id__in=collection_ids)
+
+    for collection in collections:
+        data_to_import = _get_data_to_import(
+            server_name=server_name, collection=collection
+        )
+    return len(data_to_import)
+
+
+@celery_app.task(soft_time_limit=10000)
+def import_candidate_urls_counts_from_api_all_collections_all_servers():
+    for collection in Collection.objects.all():
+        if not (
+            collection.url_count_dev == 0
+            and collection.url_count_test == 0
+            and collection.url_count_production == 0
+        ):
+            continue
+        collection_ids = [collection.id]
+        server_names = [
+            # "dev",
+            "test",
+            "production",
+        ]
+        for server_name in server_names:
+            count = import_candidate_urls_counts_from_api(server_name, collection_ids)
+            setattr(collection, f"url_count_{server_name}", count)
+            collection.save()
+
+
 @celery_app.task()
 def push_to_github_task(collection_ids):
     collections = Collection.objects.filter(id__in=collection_ids)

sde_collections/templatetags/get_server_url.py

@@ -0,0 +1,8 @@
+from django import template
+
+register = template.Library()
+
+
+@register.simple_tag
+def get_server_url(collection, server_name):
+    return collection.get_server_url(server_name)

sde_collections/urls.py

@@ -40,6 +40,11 @@
         view=views.WebappGitHubConsolidationView.as_view(),
         name="consolidate_db_and_github_configs",
     ),
+    path(
+        "url-counts/",
+        view=views.URLCountView.as_view(),
+        name="consolidate_db_and_github_configs",
+    ),
     # List all CandidateURL instances: /candidate-urls/
     # Retrieve a specific CandidateURL instance: /candidate-urls/{id}/
     # Create a new CandidateURL instance: /candidate-urls/