feat(spartan): blob sink in spartan (#11307)

.github/workflows/ci.yml

@@ -403,15 +403,24 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          # - test: reorg.test.ts
+          # - name: "reorg"
+          #   test: reorg.test.ts
           #   values: ci.yaml
           #   runner_type: 16core-tester-x86-high-memory
           #   timeout: 60
-          - test: 4epochs.test.ts
+          - name: "4epochs"
+            test: 4epochs.test.ts
             values: ci.yaml
             runner_type: 16core-tester-x86
             timeout: 40
-          # - test: gating-passive.test.ts
+          - name: "4epochs-with-blob-sink"
+            test: 4epochs.test.ts
+            values: ci.yaml
+            overrides: "blobSink.enabled=true"
+            runner_type: 16core-tester-x86
+            timeout: 40
+          # - name: "gating-passive"
+          #   test: gating-passive.test.ts
           #   values: ci.yaml
           #   runner_type: 16core-tester-x86
           #   timeout: 40
@@ -433,7 +442,7 @@ jobs:
             if ci3/test_should_run "$artifact"; then
               docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
               docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-              INSTALL_METRICS=false ./spartan/scripts/test_kind.sh "./src/spartan/${{ matrix.config.test }}" "${{ matrix.config.values }}"
+              OVERRIDES="${{ matrix.config.overrides }}" INSTALL_METRICS=false ./spartan/scripts/test_kind.sh "./src/spartan/${{ matrix.config.test }}" "${{ matrix.config.values }}"
               ci3/cache_upload_flag "$artifact"
             fi
       - name: Copy Network Logs
@@ -444,8 +453,8 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}.log
-          path: test_kind.log
+          name: kind-network-test-${{ matrix.config.name }}.log
+          path: network-test.log
 
   bb-bench:
     runs-on: ubuntu-latest

spartan/aztec-network/templates/_helpers.tpl

@@ -72,6 +72,10 @@ http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-ne
 http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }}
 {{- end -}}
 
+{{- define "aztec-network.blobSinkUrl" -}}
+http://{{ include "aztec-network.fullname" . }}-blob-sink.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.blobSink.service.nodePort }}
+{{- end -}}
+
 {{- define "aztec-network.metricsHost" -}}
 http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}

spartan/aztec-network/templates/blob-sink.yaml

@@ -0,0 +1,126 @@
+{{- if .Values.blobSink.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-blob-sink
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "aztec-network.fullname" . }}-blob-sink
+  replicas: {{ .Values.blobSink.replicas }}
+  selector:
+    matchLabels:
+      {{- include "aztec-network.selectorLabels" . | nindent 6 }}
+      app: blob-sink
+  {{- if not .Values.storage.localSsd }}
+  volumeClaimTemplates:
+    - metadata:
+        name: blob-sink-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.blobSink.dataStoreConfig.storageSize }}
+  {{- end }}
+  template:
+    metadata:
+      labels:
+        {{- include "aztec-network.selectorLabels" . | nindent 8 }}
+        app: blob-sink
+    spec:
+      {{- if .Values.storage.localSsd }}
+      {{- include "aztec-network.gcpLocalSsd" . | nindent 6 }}
+      {{- end }}
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: blob-sink
+          {{- include "aztec-network.image" . | nindent 10 }}
+          command:
+            - /bin/bash
+            - -c
+            - |
+              env && \
+              node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --blob-sink
+          startupProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.blobSink.service.nodePort }}
+            periodSeconds: {{ .Values.blobSink.startupProbe.periodSeconds }}
+            failureThreshold: {{ .Values.blobSink.startupProbe.failureThreshold }}
+          livenessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.blobSink.service.nodePort }}
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 30
+            failureThreshold: 3
+          volumeMounts:
+            - name: blob-sink-data
+              mountPath: {{ .Values.blobSink.dataStoreConfig.dataDir }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: K8S_POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OTEL_SERVICE_NAME
+              value: blob-sink
+            - name: K8S_NAMESPACE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: BLOB_SINK_PORT
+              value: "{{ .Values.blobSink.service.nodePort }}"
+            - name: LOG_LEVEL
+              value: "{{ .Values.blobSink.logLevel }}"
+            - name: LOG_JSON
+              value: "1"
+            - name: DATA_DIRECTORY
+              value: "{{ .Values.blobSink.dataStoreConfig.dataDir }}"
+            - name: DATA_STORE_MAP_SIZE_KB
+              value: "{{ .Values.blobSink.dataStoreConfig.dataStoreMapSize }}"
+            - name: USE_GCLOUD_OBSERVABILITY
+              value: "{{ .Values.telemetry.useGcloudObservability }}"
+          ports:
+            - containerPort: {{ .Values.blobSink.service.nodePort }}
+          resources:
+            {{- toYaml .Values.blobSink.resources | nindent 12 }}
+      volumes:
+      {{- if .Values.storage.localSsd }}
+        - name: blob-sink-data
+          emptyDir: {}
+      {{ else }}
+        - name: blob-sink-data
+          persistentVolumeClaim:
+            claimName: blob-sink-data
+      {{- end }}
+---
+# Headless service for StatefulSet DNS entries
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-blob-sink
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.network.public }}
+  type: LoadBalancer
+  {{- else }}
+  type: ClusterIP
+  clusterIP: None
+  {{- end }}
+  selector:
+    {{- include "aztec-network.selectorLabels" . | nindent 4 }}
+    app: blob-sink
+  ports:
+    - port: {{ .Values.blobSink.service.nodePort }}
+      name: node
+{{- end }}

spartan/aztec-network/templates/boot-node.yaml

@@ -251,6 +251,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.bootNode.service.nodePort }}
             - containerPort: {{ .Values.bootNode.service.p2pTcpPort }}

spartan/aztec-network/templates/prover-node.yaml

@@ -223,6 +223,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
             - containerPort: {{ .Values.proverNode.service.p2pTcpPort }}

spartan/aztec-network/templates/validator.yaml

@@ -237,6 +237,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.validator.service.nodePort }}
             - containerPort: {{ .Values.validator.service.p2pTcpPort }}

spartan/aztec-network/values.yaml

@@ -316,6 +316,19 @@ jobs:
   deployL1Verifier:
     enable: false
 
+blobSink:
+  enabled: false
+  replicas: 1
+  service:
+    nodePort: 5052
+  startupProbe:
+    periodSeconds: 10
+    failureThreshold: 120
+  dataStoreConfig:
+    dataDir: "/data"
+    storageSize: "8Gi"
+    dataStoreMapSize: "134217728" # 128 GB
+
 faucet:
   enabled: true
   replicas: 1

spartan/scripts/deploy_kind.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
 
 # Helper script for deploying local KIND scenarios.
+# Overrides refers to overriding values in the values yaml file
 # Usage: ./deploy_kind.sh <namespace> <values_file=default.yaml>
 # Optional environment variables:
 #   VALUES_FILE (default: "default.yaml")
 #   CHAOS_VALUES (default: "", no chaos installation)
 #   AZTEC_DOCKER_TAG (default: current git commit)
 #   INSTALL_TIMEOUT (default: 30m)
+#   OVERRIDES (default: "", no overrides)
 
 source $(git rev-parse --show-toplevel)/ci3/source
 
@@ -19,6 +21,7 @@ sepolia_deployment="${3:-false}"
 chaos_values="${CHAOS_VALUES:-}"
 aztec_docker_tag=${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}
 install_timeout=${INSTALL_TIMEOUT:-30m}
+overrides="${OVERRIDES:-}"
 
 if ! docker_has_image "aztecprotocol/aztec:$aztec_docker_tag"; then
   echo "Aztec Docker image not found. It needs to be built."
@@ -57,6 +60,17 @@ if [ -z "$chaos_values" ]; then
   kubectl delete networkchaos --all --all-namespaces 2>/dev/null || true
 fi
 
+function generate_overrides {
+  local overrides="$1"
+  if [ -n "$overrides" ]; then
+    # Split the comma-separated string into an array and generate --set arguments
+    IFS=',' read -ra OVERRIDE_ARRAY <<< "$overrides"
+    for override in "${OVERRIDE_ARRAY[@]}"; do
+      echo "--set $override"
+    done
+  fi
+}
+
 # Some configuration values are set in the eth-devnet/config/config.yaml file
 # and are used to generate the genesis.json file.
 # We need to read these values and pass them into the eth devnet create.sh script
@@ -99,6 +113,7 @@ helm upgrade --install spartan ../aztec-network \
   --create-namespace \
   "${helm_set_args[@]}" \
   --set images.aztec.image="aztecprotocol/aztec:$aztec_docker_tag" \
+  $(generate_overrides "$overrides") \
   --values "../aztec-network/values/$values_file" \
   --wait \
   --wait-for-jobs=true \

spartan/scripts/test_kind.sh

@@ -34,6 +34,8 @@ install_metrics=${INSTALL_METRICS:-true}
 use_docker=${USE_DOCKER:-true}
 sepolia_run=${SEPOLIA_RUN:-false}
 
+OVERRIDES="${OVERRIDES:-}"
+
 # Ensure we have kind context
 ../bootstrap.sh kind
 
@@ -80,7 +82,7 @@ copy_stern_to_log
 
 # uses VALUES_FILE, CHAOS_VALUES, AZTEC_DOCKER_TAG and INSTALL_TIMEOUT optional env vars
 if [ "$fresh_install" != "no-deploy" ]; then
-  ./deploy_kind.sh $namespace $values_file $sepolia_run
+  OVERRIDES="$OVERRIDES" ./deploy_kind.sh $namespace $values_file $sepolia_run
 fi
 
 # Find 4 free ports between 9000 and 10000

yarn-project/aztec/src/cli/aztec_start_action.ts

@@ -74,6 +74,9 @@ export async function aztecStart(options: any, userLog: LogFn, debugLogger: Logg
     } else if (options.proverNode) {
       const { startProverNode } = await import('./cmds/start_prover_node.js');
       ({ config } = await startProverNode(options, signalHandlers, services, userLog));
+    } else if (options.blobSink) {
+      const { startBlobSink } = await import('./cmds/start_blob_sink.js');
+      await startBlobSink(options, signalHandlers, userLog);
     } else if (options.pxe) {
       const { startPXE } = await import('./cmds/start_pxe.js');
       ({ config } = await startPXE(options, signalHandlers, services, userLog));

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -1,7 +1,7 @@
 import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/config';
 import { faucetConfigMapping } from '@aztec/aztec-faucet/config';
 import { sequencerClientConfigMappings } from '@aztec/aztec-node/config';
-import { blobSinkConfigMapping } from '@aztec/blob-sink/client';
+import { blobSinkConfigMappings } from '@aztec/blob-sink/server';
 import { botConfigMappings } from '@aztec/bot/config';
 import {
   type ConfigMapping,
@@ -259,7 +259,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       defaultValue: undefined,
       envVar: undefined,
     },
-    ...getOptions('blobSink', blobSinkConfigMapping),
+    ...getOptions('blobSink', blobSinkConfigMappings),
   ],
   'PROVER NODE': [
     {

yarn-project/aztec/src/cli/cmds/start_blob_sink.ts

@@ -0,0 +1,31 @@
+import {
+  type BlobSinkConfig,
+  blobSinkConfigMappings,
+  createBlobSinkServer,
+  getBlobSinkConfigFromEnv,
+} from '@aztec/blob-sink/server';
+import { type LogFn } from '@aztec/foundation/log';
+import { getConfigEnvVars as getTelemetryClientConfig, initTelemetryClient } from '@aztec/telemetry-client';
+
+import { extractRelevantOptions } from '../util.js';
+
+export async function startBlobSink(options: any, signalHandlers: (() => Promise<void>)[], userLog: LogFn) {
+  if (options.prover || options.node || options.sequencer || options.pxe || options.p2pBootstrap || options.txe) {
+    userLog(
+      `Starting a blob sink with --node, --sequencer, --pxe, --p2p-bootstrap, --prover or --txe is not supported.`,
+    );
+    process.exit(1);
+  }
+
+  const blobSinkConfig = {
+    ...getBlobSinkConfigFromEnv(), // get default config from env
+    ...extractRelevantOptions<BlobSinkConfig>(options, blobSinkConfigMappings, 'blobSink'), // override with command line options
+  };
+
+  const telemetry = initTelemetryClient(getTelemetryClientConfig());
+
+  const blobSink = await createBlobSinkServer(blobSinkConfig, telemetry);
+  signalHandlers.push(blobSink.stop.bind(blobSink));
+
+  await blobSink.start();
+}

yarn-project/blob-sink/src/client/http.ts

@@ -72,8 +72,7 @@ export class HttpBlobSinkClient implements BlobSinkClientInterface {
    *
    * 1. First atttempts to get blobs from a configured blob sink
    * 2. If no blob sink is configured, attempts to get blobs from a configured consensus host
-
-   * // TODO(md): blow up?
+   *
    * 3. If none configured, fails
    *
    * @param blockHash - The block hash

yarn-project/blob-sink/src/server/server.test.ts

@@ -19,6 +19,13 @@ describe('BlobSinkService', () => {
     await service.stop();
   });
 
+  describe('status', () => {
+    it('should return 200', async () => {
+      const response = await request(service.getApp()).get('/status');
+      expect(response.status).toBe(200);
+    });
+  });
+
   describe('should store and retrieve a blob sidecar', () => {
     const blockId = '0x1234';
     let blob: Blob;