[ENG-7452] introduce @expo/steps with simple config parser + CLI for …

…running exmaple (#188)

* upgrade deps elswhere

* yarn

* set up @expo/steps

* add basic implementation of BuildStep

* upgrade deps in steps

* add simple config parser + cli with example

* remove build step input/output for now

* run yarn

* update example files

* add tests

* more tests

* more tests

* export some classes

* add tests

* enforce using extensions in imports

* address PR feedback

packages/steps/.eslintrc.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../.eslintrc.json",
+  "plugins": [
+    "async-protect"
+  ],
+  "rules": {
+    "async-protect/async-suffix": "error",
+    "import/extensions": ["error", "always"]
+  }
+}

packages/steps/README.md

@@ -0,0 +1,3 @@
+# @expo/steps
+
+TBD

packages/steps/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+echo 'Removing "dist" folder...'
+rm -rf dist
+
+echo 'Compiling TypeScript to JavaScript...'
+node_modules/.bin/tsc --project tsconfig.build.json
+
+echo 'Compiling TypeScript to CommonJS JavaScript...'
+node_modules/.bin/tsc --project tsconfig.build.commonjs.json
+
+echo 'Renaming CommonJS file extensions to .cjs...'
+find dist/commonjs -type f -name '*.js' -exec bash -c 'mv "$0" "${0%.*}.cjs"' {} \;
+
+echo 'Rewriting module specifiers to .cjs...'
+find dist/commonjs -type f -name '*.cjs' -exec sed -i '' 's/require("\(\.[^"]*\)\.js")/require("\1.cjs")/g' {} \;
+
+echo 'Finished compiling'

packages/steps/cli.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+STEPS_ROOT_DIR=$( dirname "${BASH_SOURCE[0]}" )
+
+node $STEPS_ROOT_DIR/dist/commonjs/cli/cli.cjs $@ | yarn run bunyan -o short

packages/steps/examples/simple/config.yml

@@ -0,0 +1,15 @@
+build:
+  name: Foobar
+  steps:
+    - run: echo "Hi!"
+    - run:
+        name: Say HELLO
+        command: |
+          echo "H"
+          echo "E"
+          echo "L"
+          echo "L"
+          echo "O"
+    - run:
+        name: List files
+        command: ls -la

packages/steps/examples/simple/project/abc

@@ -0,0 +1 @@
+lorem ipsum

packages/steps/examples/simple/project/def

@@ -0,0 +1 @@
+lorem ipsum

packages/steps/examples/simple/project/ghi

@@ -0,0 +1 @@
+lorem ipsum

packages/steps/jest.config.cjs

@@ -0,0 +1,23 @@
+module.exports = {
+  preset: 'ts-jest/presets/default-esm',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'tsconfig.json',
+        useESM: true,
+      },
+    ],
+  },
+  testEnvironment: 'node',
+  rootDir: 'src',
+  testMatch: ['**/__tests__/*-test.ts'],
+  collectCoverage: true,
+  coverageReporters: ['json', 'lcov'],
+  coverageDirectory: '../coverage/tests/',
+  moduleNameMapper: {
+    '^(\\.\\.?/.*)\\.js$': ['$1.ts', '$0'],
+  },
+  clearMocks: true,
+  setupFilesAfterEnv: ['<rootDir>/../jest/setup-tests.ts'],
+};

packages/steps/jest/setup-tests.ts

@@ -0,0 +1,5 @@
+if (process.env.NODE_ENV !== 'test') {
+  throw new Error("NODE_ENV environment variable must be set to 'test'.");
+}
+
+// Always mock:

packages/steps/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@expo/steps",
+  "type": "module",
+  "version": "0.0.1",
+  "main": "./dist/commonjs/index.cjs",
+  "types": "./dist/esm/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/esm/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/commonjs/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "start": "yarn watch",
+    "build": "./build.sh",
+    "watch": "chokidar --initial \"src/**/*.ts\" -c \"./build.sh\"",
+    "test": "node --experimental-vm-modules --no-warnings node_modules/.bin/jest -c=jest.config.cjs --no-cache",
+    "test:watch": "yarn test --watch",
+    "clean": "rm -rf node_modules dist coverage"
+  },
+  "author": "Expo <[email protected]>",
+  "bugs": "https://github.com/expo/eas-build/issues",
+  "license": "BUSL-1.1",
+  "devDependencies": {
+    "@jest/globals": "^29.4.1",
+    "@types/jest": "^29.4.0",
+    "@types/node": "^18.11.18",
+    "bunyan": "^1.8.15",
+    "chokidar-cli": "^3.0.0",
+    "eslint-plugin-async-protect": "^3.0.0",
+    "jest": "^29.4.1",
+    "ts-jest": "^29.0.5",
+    "ts-mockito": "^2.6.1",
+    "typescript": "^4.9.5"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "dependencies": {
+    "@expo/logger": "^0.0.26",
+    "@expo/spawn-async": "^1.7.0",
+    "joi": "^17.7.0",
+    "uuid": "^9.0.0",
+    "yaml": "^2.2.1"
+  }
+}

packages/steps/src/BuildConfig.ts

@@ -0,0 +1,62 @@
+import Joi from 'joi';
+
+import { BuildConfigError } from './errors/BuildConfigError.js';
+
+export interface BuildConfig {
+  build: {
+    name?: string;
+    steps: BuildStepConfig[];
+  };
+}
+
+export type BuildStepConfig =
+  | string
+  | {
+      run:
+        | string
+        | {
+            id?: string;
+            name?: string;
+            workingDirectory?: string;
+            shell?: string;
+            command: string;
+          };
+    };
+
+export const BuildConfigSchema = Joi.object<BuildConfig>({
+  build: Joi.object({
+    name: Joi.string(),
+    steps: Joi.array()
+      .items(
+        Joi.object({
+          run: Joi.alternatives().conditional('run', {
+            is: Joi.string(),
+            then: Joi.string().required(),
+            otherwise: Joi.object({
+              id: Joi.string(),
+              name: Joi.string(),
+              workingDirectory: Joi.string(),
+              shell: Joi.string(),
+              command: Joi.string().required(),
+            })
+              .rename('working_directory', 'workingDirectory')
+              .required(),
+          }),
+        })
+      )
+      .required(),
+  }).required(),
+}).required();
+
+export function validateBuildConfig(rawConfig: object): BuildConfig {
+  const { error, value } = BuildConfigSchema.validate(rawConfig, {
+    stripUnknown: true,
+    abortEarly: false,
+  });
+  if (error) {
+    const errorMessage = error.details.map(({ message }) => message).join(', ');
+    throw new BuildConfigError(errorMessage, { cause: error });
+  } else {
+    return value;
+  }
+}

packages/steps/src/BuildConfigParser.ts

@@ -0,0 +1,57 @@
+import fs from 'fs';
+import path from 'path';
+
+import { v4 as uuidv4 } from 'uuid';
+import YAML from 'yaml';
+
+import { BuildStepConfig, validateBuildConfig } from './BuildConfig.js';
+import { BuildStep } from './BuildStep.js';
+import { BuildStepContext } from './BuildStepContext.js';
+import { BuildWorkflow } from './BuildWorkflow.js';
+
+export class BuildConfigParser {
+  private readonly configPath: string;
+
+  constructor(private readonly ctx: BuildStepContext, { configPath }: { configPath: string }) {
+    this.configPath = configPath;
+  }
+
+  public async parseAsync(): Promise<BuildWorkflow> {
+    const rawConfig = await this.readRawConfigAsync();
+    const config = validateBuildConfig(rawConfig);
+    const steps = config.build.steps.map((stepConfig) =>
+      this.createBuildStepFromConfig(stepConfig)
+    );
+    return new BuildWorkflow({ buildSteps: steps });
+  }
+
+  private async readRawConfigAsync(): Promise<any> {
+    const contents = await fs.promises.readFile(this.configPath, 'utf-8');
+    return YAML.parse(contents);
+  }
+
+  private createBuildStepFromConfig(buildStepConfig: BuildStepConfig): BuildStep {
+    if (typeof buildStepConfig === 'string') {
+      throw new Error('Not implemented yet');
+    } else if (typeof buildStepConfig.run === 'string') {
+      const command = buildStepConfig.run;
+      return new BuildStep(this.ctx, {
+        id: uuidv4(),
+        workingDirectory: this.ctx.workingDirectory,
+        command,
+      });
+    } else {
+      const { id, name, workingDirectory, shell, command } = buildStepConfig.run;
+      return new BuildStep(this.ctx, {
+        id: id ?? uuidv4(),
+        name,
+        workingDirectory:
+          workingDirectory !== undefined
+            ? path.resolve(this.ctx.workingDirectory, workingDirectory)
+            : this.ctx.workingDirectory,
+        shell,
+        command,
+      });
+    }
+  }
+}

packages/steps/src/BuildStep.ts

@@ -0,0 +1,58 @@
+import { bunyan } from '@expo/logger';
+import { v4 as uuidv4 } from 'uuid';
+
+import { BuildStepContext } from './BuildStepContext.js';
+import { getDefaultShell, getShellCommandAndArgs } from './shell/command.js';
+import { saveScriptToTemporaryFileAsync } from './shell/scripts.js';
+import { spawnAsync } from './shell/spawn.js';
+
+export class BuildStep {
+  public id: string;
+  public name?: string;
+  public command: string;
+  public workingDirectory: string;
+  public shell: string;
+
+  private readonly internalId: string;
+  private readonly logger: bunyan;
+
+  constructor(
+    private readonly ctx: BuildStepContext,
+    {
+      id,
+      name,
+      command,
+      workingDirectory,
+      shell,
+    }: {
+      id: string;
+      name?: string;
+      command: string;
+      workingDirectory: string;
+      shell?: string;
+    }
+  ) {
+    this.id = id;
+    this.name = name;
+    this.command = command;
+    this.workingDirectory = workingDirectory;
+    this.shell = shell ?? getDefaultShell();
+
+    this.internalId = uuidv4();
+    this.logger = ctx.logger.child({ buildStepInternalId: this.internalId, buildStepId: this.id });
+  }
+
+  public async executeAsync(): Promise<void> {
+    this.logger.debug(`Executing build step "${this.id}"`);
+    const scriptPath = await saveScriptToTemporaryFileAsync(this.ctx, this.id, this.command);
+    this.logger.debug(`Saved script to ${scriptPath}`);
+    const { command, args } = getShellCommandAndArgs(this.shell, scriptPath);
+    this.logger.debug(
+      `Executing script: ${command}${args !== undefined ? ` ${args.join(' ')}` : ''}`
+    );
+    await spawnAsync(command, args ?? [], {
+      cwd: this.workingDirectory,
+      logger: this.logger,
+    });
+  }
+}

packages/steps/src/BuildStepContext.ts

@@ -0,0 +1,19 @@
+import os from 'os';
+import path from 'path';
+
+import { bunyan } from '@expo/logger';
+
+export class BuildStepContext {
+  public readonly baseWorkingDirectory: string;
+  public readonly workingDirectory: string;
+
+  constructor(
+    public readonly buildId: string,
+    public readonly logger: bunyan,
+    public readonly skipCleanup: boolean,
+    workingDirectory?: string
+  ) {
+    this.baseWorkingDirectory = path.join(os.tmpdir(), 'eas-build', buildId);
+    this.workingDirectory = workingDirectory ?? path.join(this.baseWorkingDirectory, 'project');
+  }
+}

packages/steps/src/BuildWorkflow.ts

@@ -0,0 +1,15 @@
+import { BuildStep } from './BuildStep.js';
+
+export class BuildWorkflow {
+  public readonly buildSteps: BuildStep[];
+
+  constructor({ buildSteps }: { buildSteps: BuildStep[] }) {
+    this.buildSteps = buildSteps;
+  }
+
+  public async executeAsync(): Promise<void> {
+    for (const step of this.buildSteps) {
+      await step.executeAsync();
+    }
+  }
+}

packages/steps/src/__tests__/BuildConfig-test.ts

@@ -0,0 +1,30 @@
+import { validateBuildConfig } from '../BuildConfig.js';
+import { BuildConfigError } from '../errors/BuildConfigError.js';
+
+describe(validateBuildConfig, () => {
+  test('can throw BuildConfigError', () => {
+    const buildConfig = {};
+
+    expect(() => {
+      validateBuildConfig(buildConfig);
+    }).toThrowError(BuildConfigError);
+  });
+
+  describe('steps', () => {
+    test('command is required', () => {
+      const buildConfig = {
+        build: {
+          steps: [
+            {
+              run: {},
+            },
+          ],
+        },
+      };
+
+      expect(() => {
+        validateBuildConfig(buildConfig);
+      }).toThrowError(/".*\.command" is required/);
+    });
+  });
+});