feat(compartment-mapper): Make script or functor bundles

packages/compartment-mapper/NEWS.md

@@ -2,6 +2,28 @@ User-visible changes to `@endo/compartment-mapper`:
 
 # Next release
 
+- Divides the role of `makeBundle` into `makeScript` and `makeFunctor`.
+  The new `makeScript` replaces `makeBundle` without breaking changes,
+  producing a JavaScript string that is suitable as a `<script>` tag in a web
+  page.
+- The new `makeFunctor` produces a JavaScript string that, when evaluated,
+  produces a partially applied function, so the caller can provide runtime
+  options.
+- Both `makeScript` and `makeFunctor` now accept `format`, `useEvaluate` and
+  `sourceUrlPrefix` options.
+- The functor produced by `makeFunctor` now accepts `evaluate`, `require`, and
+  `sourceUrlPrefix` runtime options.
+- Both `makeScript` and `makeFunctor` now accept a `format` option.
+  Specifiying the `"cjs"` format allows the bundle to exit to the host's
+  CommonJS `require` for host modules.
+- Adds `sourceDirname` to compartment descriptors in the compartment maps
+  generated by `mapNodeModules` and uses these to provide better source URL
+  comments for bundles generated by `makeScript` and `makeFunctor`, by default.
+
+These changes collectively allow us to replace the implementation of
+`nestedEvaluate` and `getExports` formats in `@endo/bundle-source`, including
+the preservation of useful line numbers and file names in stack traces.
+
 - `mapNodeModules`, `importLocation` and `loadLocation` now accept a `log`
   option for users to define a custom logging function. As of this writing,
   _only `mapNodeModules`_ will potentially call this function if provided.

packages/compartment-mapper/README.md

@@ -137,6 +137,173 @@ Use `parseArchive` to construct a runner from the bytes of an archive.
 `loadArchive` and `parseArchive` do not run the archived application,
 so they can be used to safely check its hash.
 
+# Script bundles
+
+From `@endo/compartment-mapper/script.js`, the `makeScript` function is similar
+to `makeArchive` but generates a string of JavaScript suitable for `eval` or
+embedding in a web page with a `<script>`.
+Endo uses this "bundle" format to bootstrap an environment up to the point it
+can call `importArchive`, so bundles are at least suitable for creating a
+script that subsumes `ses`, `@endo/compartment-mapper/import-archive.js`, and
+other parts of Endo, but is not as feature-complete as `importArchive`.
+
+```js
+import url from "url";
+import fs from "fs";
+import { makeScript } from "@endo/compartment-mapper/script.js";
+import { makeReadPowers } from "@endo/compartment-mapper/node-powers.js";
+const readPowers = makeReadPowers({ fs, url });
+const options = {}; // if any
+const script = await makeScript(readPowers, moduleSpecifier, options);
+```
+
+The script is suitable for evaluating as a script in a web environment.
+The script is in UTF-8 format and uses non-ASCII characters, so may require
+headers or tags to specify the encoding.
+
+```html
+<meta charset="utf-8">
+<script src="script.js"></script>
+```
+
+Evaluation of `script` returns the emulated exports namespace of the entry
+module.
+
+```js
+const script = await makeScript(readPowers, moduleSpecifier, options);
+
+// This one weird trick evaluates your script in global scope instead of
+// lexical scope.
+const globalEval = eval;
+const moduleExports = globalEval(script);
+```
+
+Scripts can include ESM, CJS, and JSON modules, but no other module languages
+like bytes or text.
+
+> [!WARNING]
+> Scripts do not support [live
+> bindings](https://developer.mozilla.org/en-US/docs/Glossary/Binding), dynamic
+> `import`, or `import.meta`.
+> Scripts do not isolate modules to a compartment.
+
+`makeScript` accepts all the options of `makeArchive` and:
+
+- `sourceUrlPrefix` (string, default `""`):
+  Specifies a prefix to occur on each module's `sourceURL` comment, as injected
+  at runtime.
+  Should generally end with `/` if non-empty.
+  This can improve stack traces.
+- `format` (`"cjs"` or `undefined`, default `undefined`):
+  By default, `makeBundle` generates a bundle that can be evaluated in any
+  context.
+  By specifying `"cjs"`, the bundle can assume there is a host CommonJS
+  `require` function available for resolving modules that exit the bundle.
+  The default is `require` on `globalThis`.
+  The `require` function can be overridden with a curried runtime option.
+- `useEvaluate` (boolean, default `false`):
+  Disabled by default, for bundles that may be embedded on a web page with a
+  `no-unsafe-eval` Content Security Policy.
+  Enable for any environment that can use `eval` or other suitable evaluator
+  (like a Hardened JavaScript `Compartment`).
+
+  By default and when `useEvaluate` is explicitly `false`, the text of a module
+  includes an array of module evaluator functions.
+
+  > [!WARNING]
+  > Example is illustrative and neither a compatibility guarantee nor even
+  > precise.
+
+  ```js
+  (modules => options => {
+    /* ...linker runtime... */
+    for (const module of modules) {
+      module(/* linking convention */);
+    }
+  )([
+  // 1. bundle ./dependency.js
+  function () { /* ... */ },
+  // 2. bundle ./dependent.js
+  function () { /* ... */ },
+  ])(/* runtime options */)
+  ```
+
+  Each of these functions is generated by [Endo's emulation of a JavaScript
+  `ModuleSource`
+  constructor](https://github.com/endojs/endo/blob/master/packages/module-source/DESIGN.md),
+  which we use elsewhere in the Compartment Mapper to emulate Compartment
+  module systems at runtime, as in the Compartment Mapper's own `importArchive`.
+
+  With `useEvaluate`, the script instead embeds the text for each module as a
+  string, along with a package-relative source URL, and uses an `eval` function
+  to produce the corresponding `function`.
+
+  ```js
+  (modules => options => {
+    /* ...linker runtime... */
+    for (const [module, sourceURL] of modules) {
+      evalWithSourceURL(module, sourceURL)(/* linking convention */);
+    }
+  )([
+  // 1. bundle ./dependency.js
+  ["(function () { /* ... */ })", "bundle/dependency.js"],
+  // 2. bundle ./dependent.js
+  ["(function () { /* ... */ })", "bundle/dependent.js"],
+  ])(/* runtime options */)
+  ```
+
+  With `useEvaluate`, the bundle will instead capture a string for
+  each module function and use an indirect `eval` to revive them.
+  This can make the file locations and line numbers in stack traces more
+  useful.
+
+From `@endo/compartment-mapper/script-lite.js`, the `makeScriptFromMap` takes
+a compartment map, like that generated by `mapNodeModules` in
+`@endo/compartment-mapper/node-modules.js` instead of the entry module's
+location.
+The `-lite.js` modules, in general, do not entrain a specific compartment
+mapper.
+
+# Functor bundles
+
+From `@endo/compartment-mapper/functor.js`, the `makeFunctor` function is similar
+to `makeScript` but generates a string of JavaScript suitable for `eval` but *not*
+suitable for embedding as a script. But, the completion value of the script
+is a function that accepts runtime options and returns the entry module's emulated
+module exports namespace, adding a level of indirection.
+
+In this example, we use a Hardened JavaScript `Compartment` to confine the
+execution of the functor and its modules.
+
+```js
+const functorScript = await makeFunctor(readPowers, moduleSpecifier, options);
+const compartment = new Compartment();
+const moduleExports = compartment.evaluate(functorScript)({
+  require,
+  evaluate: compartment.evaluate,
+  sourceUrlPrefix: 'file:///Users/you/project/',
+});
+```
+
+The functor runtime options include:
+
+- `evaluate`: for functors made with `useEvaluate`,
+  specifies a function to use to evaluate each module.
+  The default evaluator is indirect `eval`.
+- `require`: for functors made with `format` of `"cjs"`, provides the behavior
+  for `require` calls that exit the bundle to the host environment.
+  Defaults to the `require` in lexical scope.
+- `sourceUrlPrefix`: specifies a prefix to occur on each module's `sourceURL` comment,
+  as injected at runtime.
+  Overrides the `sourceUrlPrefix` provided to `makeFunctor`, if any.
+
+From `@endo/compartment-mapper/functor-lite.js`, the `makeFunctorFromMap` takes
+a compartment map, like that generated by `mapNodeModules` in
+`@endo/compartment-mapper/node-modules.js` instead of the entry module's
+location.
+The `-lite.js` modules, in general, do not entrain a specific compartment
+mapper.
+
 # Package Descriptors
 
 The compartment mapper uses [Compartments], one for each Node.js package your
@@ -598,7 +765,6 @@ The shape of the `policy` object is based on `policy.json` from LavaMoat. MetaMa
 > policy.json.
 > Policy generation may be ported to Endo.
 
-
   [LavaMoat]: https://github.com/LavaMoat/lavamoat
   [Compartments]: ../ses/README.md#compartment
   [Policy Demo]: ./demo/policy/README.md

packages/compartment-mapper/bundle.js

@@ -1,4 +1,7 @@
 // eslint-disable-next-line import/export -- just types
 export * from './src/types-external.js';
 
-export { makeBundle, writeBundle } from './src/bundle.js';
+export {
+  makeScript as makeBundle,
+  writeScript as writeBundle,
+} from './src/bundle.js';

packages/compartment-mapper/functor-lite.js

@@ -0,0 +1,4 @@
+// eslint-disable-next-line import/export -- just types
+export * from './src/types-external.js';
+
+export { makeFunctorFromMap } from './src/bundle-lite.js';

packages/compartment-mapper/functor.js

@@ -0,0 +1,4 @@
+// eslint-disable-next-line import/export -- just types
+export * from './src/types-external.js';
+
+export { makeFunctor } from './src/bundle.js';

packages/compartment-mapper/index.js

@@ -16,4 +16,7 @@ export {
 } from './src/import-archive.js';
 export { search } from './src/search.js';
 export { compartmentMapForNodeModules } from './src/node-modules.js';
-export { makeBundle, writeBundle } from './src/bundle.js';
+export {
+  makeScript as makeBundle,
+  writeScript as writeBundle,
+} from './src/bundle.js';

packages/compartment-mapper/package.json

@@ -39,6 +39,10 @@
     },
     "./import-archive-all-parsers.js": "./import-archive-all-parsers.js",
     "./bundle.js": "./bundle.js",
+    "./functor.js": "./functor.js",
+    "./functor-lite.js": "./functor-lite.js",
+    "./script.js": "./script.js",
+    "./script-lite.js": "./script-lite.js",
     "./node-powers.js": "./node-powers.js",
     "./node-modules.js": "./node-modules.js",
     "./package.json": "./package.json"

packages/compartment-mapper/script-lite.js

@@ -0,0 +1,4 @@
+// eslint-disable-next-line import/export -- just types
+export * from './src/types-external.js';
+
+export { makeScriptFromMap } from './src/bundle-lite.js';

packages/compartment-mapper/script.js

@@ -0,0 +1,4 @@
+// eslint-disable-next-line import/export -- just types
+export * from './src/types-external.js';
+
+export { makeScript } from './src/bundle.js';

packages/compartment-mapper/src/archive-lite.js

@@ -176,7 +176,7 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => {
     searchSuffixes,
     entryCompartmentName,
     entryModuleSpecifier,
-    exitModuleImportHook: consolidatedExitModuleImportHook,
+    importHook: consolidatedExitModuleImportHook,
     sourceMapHook,
   });
   // Induce importHook to record all the necessary modules to import the given module specifier.