Add RE2 supervisor (#1769)

* add new core server only export

* exit after run if warm starts disabled

* add supervisor

Author: nicktrn

apps/coordinator/src/checkpointer.ts

@@ -1,5 +1,5 @@
 import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
-import { testDockerCheckpoint } from "@trigger.dev/core/v3/checkpoints";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
 import { nanoid } from "nanoid";
 import fs from "node:fs/promises";
 import { ChaosMonkey } from "./chaosMonkey";

apps/supervisor/.env.example

@@ -0,0 +1,18 @@
+# This needs to match the token of the worker group you want to connect to
+TRIGGER_WORKER_TOKEN=
+
+# This needs to match the MANAGED_WORKER_SECRET env var on the webapp
+MANAGED_WORKER_SECRET=managed-secret
+
+# Point this at the webapp in prod
+TRIGGER_API_URL=http://localhost:3030
+
+# Point this at the OTel collector in prod
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3030/otel
+# Use this on macOS
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:3030/otel
+
+# Optional settings
+DEBUG=1
+ENFORCE_MACHINE_PRESETS=1
+TRIGGER_DEQUEUE_INTERVAL_MS=1000

apps/supervisor/.nvmrc

@@ -0,0 +1 @@
+v22.12.0

apps/supervisor/README.md

@@ -0,0 +1,49 @@
+# Supervisor
+
+## Dev setup
+
+1. Create a worker group
+
+```sh
+api_url=http://localhost:3030
+wg_name=my-worker
+
+# edit these
+admin_pat=tr_pat_...
+project_id=clsw6q8wz...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"name\": \"$wg_name\",
+        \"makeDefault\": true,
+        \"projectId\": \"$project_id\"
+    }"
+```
+
+2. Create `.env` and set the worker token
+
+```sh
+cp .env.example .env
+
+# Then edit your .env and set this to the token.plaintext value
+TRIGGER_WORKER_TOKEN=tr_wgt_...
+```
+
+3. Start the supervisor
+
+```sh
+pnpm dev
+```
+
+4. Build CLI, then deploy a reference project
+
+```sh
+pnpm exec trigger deploy --self-hosted
+
+# The additional network flag is required on linux
+pnpm exec trigger deploy --self-hosted --network host
+```

apps/supervisor/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "supervisor",
+  "private": true,
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "dev": "tsx --experimental-sqlite --require dotenv/config --watch src/index.ts",
+    "start": "node --experimental-sqlite dist/index.js",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@kubernetes/client-node": "^1.0.0",
+    "@trigger.dev/core": "workspace:*",
+    "dockerode": "^4.0.3",
+    "nanoid": "^5.0.9",
+    "socket.io": "4.7.4",
+    "std-env": "^3.8.0",
+    "tinyexec": "^0.3.1",
+    "zod": "3.23.8"
+  },
+  "devDependencies": {
+    "@types/dockerode": "^3.3.33",
+    "docker-api-ts": "^0.2.2"
+  }
+}

apps/supervisor/src/clients/kubernetes.ts

@@ -0,0 +1,39 @@
+import * as k8s from "@kubernetes/client-node";
+import { assertExhaustive } from "@trigger.dev/core/utils";
+
+export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+
+export function createK8sApi() {
+  const kubeConfig = getKubeConfig();
+
+  const api = {
+    core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+    batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+    apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+  };
+
+  return api;
+}
+
+export type K8sApi = ReturnType<typeof createK8sApi>;
+
+function getKubeConfig() {
+  console.log("getKubeConfig()", { RUNTIME_ENV });
+
+  const kubeConfig = new k8s.KubeConfig();
+
+  switch (RUNTIME_ENV) {
+    case "local":
+      kubeConfig.loadFromDefault();
+      break;
+    case "kubernetes":
+      kubeConfig.loadFromCluster();
+      break;
+    default:
+      assertExhaustive(RUNTIME_ENV);
+  }
+
+  return kubeConfig;
+}
+
+export { k8s };

apps/supervisor/src/env.ts

@@ -0,0 +1,30 @@
+import { randomUUID } from "crypto";
+import { env as stdEnv } from "std-env";
+import { z } from "zod";
+import { getDockerHostDomain } from "./util.js";
+
+const Env = z.object({
+  // This will come from `status.hostIP` in k8s
+  WORKER_HOST_IP: z.string().default(getDockerHostDomain()),
+  TRIGGER_API_URL: z.string().url(),
+  TRIGGER_WORKER_TOKEN: z.string(),
+  // This will come from `spec.nodeName` in k8s
+  TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+  MANAGED_WORKER_SECRET: z.string(),
+  TRIGGER_WORKLOAD_API_PORT: z.coerce.number().default(8020),
+  TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020),
+  TRIGGER_WARM_START_URL: z.string().optional(),
+  TRIGGER_CHECKPOINT_URL: z.string().optional(),
+  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(1000),
+
+  // Used by the workload manager, e.g docker/k8s
+  DOCKER_NETWORK: z.string().default("host"),
+  OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
+  ENFORCE_MACHINE_PRESETS: z.coerce.boolean().default(false),
+
+  // Used by the resource monitor
+  OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
+  OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+});
+
+export const env = Env.parse(stdEnv);