DM-48589: Set up preliminary DP1 Butler in idfdev

applications/butler/README.md

@@ -18,10 +18,12 @@ Server for Butler data abstraction service
 | config.additionalS3EndpointUrls | object | No additional URLs | Endpoint URLs for additional S3 services used by the Butler, as a mapping from profile name to URL. |
 | config.dp02ClientServerIsDefault | bool | `false` | True if the 'dp02' Butler repository alias should use client/server Butler.  False if it should use DirectButler. |
 | config.dp02PostgresUri | string | No configuration file for DP02 will be generated. | Postgres connection string pointing to the registry database hosting Data Preview 0.2 data. |
+| config.dp1PostgresUri | string | No configuration file for DP1 will be generated. | Postgres connection string pointing to the registry database hosting Data Preview 1 data. |
 | config.pathPrefix | string | `"/api/butler"` | The prefix of the path portion of the URL where the Butler service will be exposed.  For example, if the service should be exposed at `https://data.lsst.cloud/api/butler`, this should be set to `/api/butler` |
 | config.pguser | string | Use values specified in per-repository Butler config files. | Postgres username used to connect to the Butler DB |
 | config.repositories | object | `{}` | Mapping from Butler repository label to Butler configuration URI for repositories which will be hosted by this server. |
 | config.s3EndpointUrl | string | `""` | URL for the primary S3 service where files for datasets are stored by Butler. |
+| config.shareNubladoSecrets | bool | `true` | If true, borrow the S3 and Postgres secrets set up in Nublado for end-users.  Otherwise, use secrets specifically set up for the Butler server. |
 | global.baseUrl | string | Set by Argo CD | Base URL for the environment |
 | global.host | string | Set by Argo CD | Host name for ingress |
 | global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |

applications/butler/secrets.yaml

@@ -5,16 +5,15 @@
   copy:
     application: nublado
     key: "aws-credentials.ini"
-"butler-gcs-creds.json":
-  description: >-
-    Google Cloud Storage credentials to the Butler data store in the native
-    Google syntax, containing the private asymmetric key.
-  copy:
-    application: nublado
-    key: "butler-gcs-idf-creds.json"
+    if: config.shareNubladoSecrets
+  onepassword:
+    encoded: true
 "postgres-credentials.txt":
   description: >-
     PostgreSQL credentials in its pgpass format for the Butler database.
   copy:
     application: nublado
     key: "postgres-credentials.txt"
+    if: config.shareNubladoSecrets
+  onepassword:
+    encoded: true

applications/butler/templates/configmap-private.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: butler-private
+data:
+  dp1.yaml: |
+    datastore:
+      cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
+      records:
+        table: file_datastore_records
+      root: s3://gcs@butler-us-central1-dp1/
+    registry:
+      db: {{ .Values.config.dp1PostgresUri }}
+      managers:
+        attributes: lsst.daf.butler.registry.attributes.DefaultButlerAttributeManager
+        collections: lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager
+        datasets: lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID
+        datastores: lsst.daf.butler.registry.bridge.monolithic.MonolithicDatastoreRegistryBridgeManager
+        dimensions: lsst.daf.butler.registry.dimensions.static.StaticDimensionRecordStorageManager
+        opaque: lsst.daf.butler.registry.opaque.ByNameOpaqueTableStorageManager
+      namespace: prelim1

applications/butler/templates/configmap.yaml → applications/butler/templates/configmap-public.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: butler
+  name: butler-public
 data:
 # WARNING! The files in this configmap are served publicly on the internet via unauthenticated HTTP.
 # DO NOT PUT ANY SENSITIVE INFORMATION IN THESE FILES.
@@ -81,4 +81,5 @@ data:
     {{- end }}
     dp02-direct: {{ $dp02Direct }}
     dp02-remote: {{ $dp02Remote }}
+    dp1: "{{.Values.global.baseUrl}}{{.Values.config.pathPrefix}}/repo/dp1/butler.yaml"
 {{- end }}

applications/butler/templates/deployment.yaml

@@ -15,7 +15,7 @@ spec:
     metadata:
       annotations:
         # Force the pod to restart when the config maps are updated.
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/config: {{ print (include (print $.Template.BasePath "/configmap-public.yaml") .) (include (print $.Template.BasePath "/configmap-private.yaml") .)| sha256sum }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -49,8 +49,6 @@ spec:
               value: "/opt/lsst/butler/secrets/aws-credentials.ini"
             - name: PGPASSFILE
               value: "/opt/lsst/butler/secrets/postgres-credentials.txt"
-            - name: GOOGLE_APPLICATION_CREDENTIALS
-              value: "/opt/lsst/butler/secrets/butler-gcs-creds.json"
             - name: S3_ENDPOINT_URL
               value: {{ .Values.config.s3EndpointUrl | quote }}
             {{- range $name, $url := .Values.config.additionalS3EndpointUrls }}
@@ -59,13 +57,13 @@ spec:
             {{ end }}
             - name: DAF_BUTLER_REPOSITORIES
               value: {{ .Values.config.repositories | toJson | quote }}
-            # Serve the configuration files generated by configmap.yaml via
-            # unauthenticated HTTP, for consumption by Butler users in RSP
+            # Serve the configuration files generated by configmap-public.yaml
+            # via unauthenticated HTTP, for consumption by Butler users in RSP
             # notebook lab containers.
             # (There isn't another obvious place for these configuration files
             # to live, so we just have Butler server host them.)
             - name: DAF_BUTLER_SERVER_STATIC_FILES_PATH
-              value: "/opt/lsst/butler/config"
+              value: "/opt/lsst/butler/public/config"
             {{ if .Values.config.pguser }}
             - name: PGUSER
               value: {{ .Values.config.pguser | quote }}
@@ -74,8 +72,12 @@ spec:
             - name: "butler-secrets"
               mountPath: "/opt/lsst/butler/secrets"
               readOnly: true
-            # Mount configuration files generated in configmap.yaml.
-            - name: config
+            # Mount configuration files generated in configmap-public.yaml.
+            - name: config-public
+              mountPath: "/opt/lsst/butler/public/config"
+              readOnly: true
+            # Mount configuration files generated in configmap-private.yaml.
+            - name: config-private
               mountPath: "/opt/lsst/butler/config"
               readOnly: true
       volumes:
@@ -94,9 +96,12 @@ spec:
         # Butler application.
         - name: "butler-secrets"
           emptyDir: {}
-        - name: config
+        - name: config-public
+          configMap:
+            name: butler-public
+        - name: config-private
           configMap:
-            name: butler
+            name: butler-private
       initContainers:
         # To deal with the Postgres file permission issued mentioned above,
         # copy the secrets from butler-secrets-raw to butler-secrets.

applications/butler/templates/ingress-anonymous.yaml

@@ -29,14 +29,16 @@ template:
             # configuration file is loaded.
             {{- range $repositoryLabel, $unused := .Values.config.repositories }}
             - path: "{{ $.Values.config.pathPrefix }}/repo/{{ $repositoryLabel }}/butler.yaml"
-              pathType: "Exact"
+              # These should be 'pathType: Exact', but newer versions of Kubernetes don't allow periods
+              # in the path.  See https://github.com/kubernetes/ingress-nginx/issues/11176
+              pathType: "ImplementationSpecific"
               backend:
                 service:
                   name: "butler"
                   port:
                     number: 8080
             - path: "{{ $.Values.config.pathPrefix }}/repo/{{ $repositoryLabel }}/butler.json"
-              pathType: "Exact"
+              pathType: "ImplementationSpecific"
               backend:
                 service:
                   name: "butler"

applications/butler/values-idfdev.yaml

@@ -5,6 +5,9 @@ config:
   dp02ClientServerIsDefault: true
   # butler-registry-dp02-dev Google Cloud SQL instance in science-platform-dev
   dp02PostgresUri: postgresql://[email protected]:5432/dp02
+  dp1PostgresUri: postgresql://[email protected]:5432/dp1
   s3EndpointUrl: "https://storage.googleapis.com"
   repositories:
-    dp02: "file:///opt/lsst/butler/config/dp02.yaml"
+    dp02: "file:///opt/lsst/butler/public/config/dp02.yaml"
+    dp1: "file:///opt/lsst/butler/config/dp1.yaml"
+  shareNubladoSecrets: false

applications/butler/values-idfint.yaml

@@ -5,4 +5,4 @@ config:
   dp02PostgresUri: postgresql://[email protected]:5432/dp02
   s3EndpointUrl: "https://storage.googleapis.com"
   repositories:
-    dp02: "file:///opt/lsst/butler/config/dp02.yaml"
+    dp02: "file:///opt/lsst/butler/public/config/dp02.yaml"

applications/butler/values-idfprod.yaml

@@ -5,4 +5,4 @@ config:
   dp02PostgresUri: postgresql://[email protected]:5432/dp02
   s3EndpointUrl: "https://storage.googleapis.com"
   repositories:
-    dp02: "file:///opt/lsst/butler/config/dp02.yaml"
+    dp02: "file:///opt/lsst/butler/public/config/dp02.yaml"

applications/butler/values.yaml

@@ -96,6 +96,11 @@ config:
   # @default -- No configuration file for DP02 will be generated.
   dp02PostgresUri: ""
 
+  # -- Postgres connection string pointing to the registry database hosting
+  # Data Preview 1 data.
+  # @default -- No configuration file for DP1 will be generated.
+  dp1PostgresUri: ""
+
   # -- True if the 'dp02' Butler repository alias should use client/server
   # Butler.  False if it should use DirectButler.
   dp02ClientServerIsDefault: false
@@ -116,3 +121,8 @@ config:
   # be exposed.  For example, if the service should be exposed at
   # `https://data.lsst.cloud/api/butler`, this should be set to `/api/butler`
   pathPrefix: "/api/butler"
+
+  # -- If true, borrow the S3 and Postgres secrets set up in Nublado for
+  # end-users.  Otherwise, use secrets specifically set up for the Butler
+  # server.
+  shareNubladoSecrets: true

environments/values-idfdev.yaml

@@ -3,6 +3,7 @@ fqdn: "data-dev.lsst.cloud"
 appOfAppsName: "science-platform"
 butlerServerRepositories:
   dp02: "https://data-dev.lsst.cloud/api/butler/repo/dp02/butler.yaml"
+  dp1: "https://data-dev.lsst.cloud/api/butler/repo/dp1/butler.yaml"
 gcp:
   projectId: "science-platform-dev-7696"
   region: "us-central1"