diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index 065ed976cb03f..7f6b22238fe52 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -325,13 +325,13 @@ }, { "name": "Powerpack Features", - "path": "/features/powerpack-features", - "id": "powerpack-features", + "path": "/features/powerpack", + "id": "powerpack", "isExternal": false, "children": [ { "name": "Run Language-Agnostic Conformance Rules", - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "id": "conformance", "isExternal": false, "children": [], @@ -339,7 +339,7 @@ }, { "name": "Define Code Ownership at the Project Level", - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "id": "owners", "isExternal": false, "children": [], @@ -347,7 +347,7 @@ }, { "name": "Change the Remote Cache Storage Location", - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "id": "custom-caching", "isExternal": false, "children": [], @@ -474,13 +474,13 @@ }, { "name": "Powerpack Features", - "path": "/features/powerpack-features", - "id": "powerpack-features", + "path": "/features/powerpack", + "id": "powerpack", "isExternal": false, "children": [ { "name": "Run Language-Agnostic Conformance Rules", - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "id": "conformance", "isExternal": false, "children": [], @@ -488,7 +488,7 @@ }, { "name": "Define Code Ownership at the Project Level", - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "id": "owners", "isExternal": false, "children": [], @@ -496,7 +496,7 @@ }, { "name": "Change the Remote Cache Storage Location", - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "id": "custom-caching", "isExternal": false, "children": [], @@ -507,7 +507,7 @@ }, { "name": "Run Language-Agnostic Conformance Rules", - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "id": "conformance", "isExternal": false, "children": [], @@ -515,7 +515,7 @@ }, { "name": "Define Code Ownership at the Project Level", - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "id": "owners", "isExternal": false, "children": [], @@ -523,7 +523,7 @@ }, { "name": "Change the Remote Cache Storage Location", - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "id": "custom-caching", "isExternal": false, "children": [], @@ -4876,6 +4876,14 @@ "children": [], "disableCollapsible": false }, + { + "name": "Custom Task Runners", + "path": "/deprecated/custom-task-runners", + "id": "custom-task-runners", + "isExternal": false, + "children": [], + "disableCollapsible": false + }, { "name": "Workspace Executors", "path": "/deprecated/workspace-executors", @@ -4975,6 +4983,14 @@ "children": [], "disableCollapsible": false }, + { + "name": "Custom Task Runners", + "path": "/deprecated/custom-task-runners", + "id": "custom-task-runners", + "isExternal": false, + "children": [], + "disableCollapsible": false + }, { "name": "Workspace Executors", "path": "/deprecated/workspace-executors", diff --git a/docs/generated/manifests/nx.json b/docs/generated/manifests/nx.json index 709235675084d..0da64f8865203 100644 --- a/docs/generated/manifests/nx.json +++ b/docs/generated/manifests/nx.json @@ -439,7 +439,7 @@ "tags": ["nx-release"] }, { - "id": "powerpack-features", + "id": "powerpack", "name": "Powerpack Features", "description": "Features of Nx that are available with a powerpack license.", "mediaImage": "", @@ -453,7 +453,7 @@ "file": "shared/features/powerpack/conformance", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "tags": ["conformance"] }, { @@ -464,7 +464,7 @@ "file": "shared/features/powerpack/owners", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "tags": ["owners"] }, { @@ -475,12 +475,12 @@ "file": "shared/features/powerpack/custom-caching", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "tags": ["custom-caching"] } ], "isExternal": false, - "path": "/features/powerpack-features", + "path": "/features/powerpack", "tags": [] }, { @@ -643,8 +643,8 @@ "path": "/features/manage-releases", "tags": ["nx-release"] }, - "/features/powerpack-features": { - "id": "powerpack-features", + "/features/powerpack": { + "id": "powerpack", "name": "Powerpack Features", "description": "Features of Nx that are available with a powerpack license.", "mediaImage": "", @@ -658,7 +658,7 @@ "file": "shared/features/powerpack/conformance", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "tags": ["conformance"] }, { @@ -669,7 +669,7 @@ "file": "shared/features/powerpack/owners", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "tags": ["owners"] }, { @@ -680,15 +680,15 @@ "file": "shared/features/powerpack/custom-caching", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "tags": ["custom-caching"] } ], "isExternal": false, - "path": "/features/powerpack-features", + "path": "/features/powerpack", "tags": [] }, - "/features/powerpack-features/conformance": { + "/features/powerpack/conformance": { "id": "conformance", "name": "Run Language-Agnostic Conformance Rules", "description": "Conformance", @@ -696,10 +696,10 @@ "file": "shared/features/powerpack/conformance", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/conformance", + "path": "/features/powerpack/conformance", "tags": ["conformance"] }, - "/features/powerpack-features/owners": { + "/features/powerpack/owners": { "id": "owners", "name": "Define Code Ownership at the Project Level", "description": "Owners", @@ -707,10 +707,10 @@ "file": "shared/features/powerpack/owners", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/owners", + "path": "/features/powerpack/owners", "tags": ["owners"] }, - "/features/powerpack-features/custom-caching": { + "/features/powerpack/custom-caching": { "id": "custom-caching", "name": "Change the Remote Cache Storage Location", "description": "Custom Caching", @@ -718,7 +718,7 @@ "file": "shared/features/powerpack/custom-caching", "itemList": [], "isExternal": false, - "path": "/features/powerpack-features/custom-caching", + "path": "/features/powerpack/custom-caching", "tags": ["custom-caching"] }, "/features/ci-features": { @@ -6677,6 +6677,17 @@ "path": "/deprecated/workspace-generators", "tags": [] }, + { + "id": "custom-task-runners", + "name": "Custom Task Runners", + "description": "", + "mediaImage": "", + "file": "shared/deprecated/custom-task-runners", + "itemList": [], + "isExternal": false, + "path": "/deprecated/custom-task-runners", + "tags": [] + }, { "id": "workspace-executors", "name": "Workspace Executors", @@ -6814,6 +6825,17 @@ "path": "/deprecated/workspace-generators", "tags": [] }, + "/deprecated/custom-task-runners": { + "id": "custom-task-runners", + "name": "Custom Task Runners", + "description": "", + "mediaImage": "", + "file": "shared/deprecated/custom-task-runners", + "itemList": [], + "isExternal": false, + "path": "/deprecated/custom-task-runners", + "tags": [] + }, "/deprecated/workspace-executors": { "id": "workspace-executors", "name": "Workspace Executors", diff --git a/docs/generated/manifests/tags.json b/docs/generated/manifests/tags.json index 4a996615b45da..d89ae8cdb2689 100644 --- a/docs/generated/manifests/tags.json +++ b/docs/generated/manifests/tags.json @@ -581,7 +581,7 @@ "file": "shared/features/powerpack/conformance", "id": "conformance", "name": "Run Language-Agnostic Conformance Rules", - "path": "/features/powerpack-features/conformance" + "path": "/features/powerpack/conformance" } ], "owners": [ @@ -590,7 +590,7 @@ "file": "shared/features/powerpack/owners", "id": "owners", "name": "Define Code Ownership at the Project Level", - "path": "/features/powerpack-features/owners" + "path": "/features/powerpack/owners" } ], "custom-caching": [ @@ -599,7 +599,7 @@ "file": "shared/features/powerpack/custom-caching", "id": "custom-caching", "name": "Change the Remote Cache Storage Location", - "path": "/features/powerpack-features/custom-caching" + "path": "/features/powerpack/custom-caching" } ], "intro": [ diff --git a/docs/map.json b/docs/map.json index 60857fe58436f..a73cb84ad906f 100644 --- a/docs/map.json +++ b/docs/map.json @@ -133,7 +133,7 @@ }, { "name": "Powerpack Features", - "id": "powerpack-features", + "id": "powerpack", "description": "Features of Nx that are available with a powerpack license.", "itemList": [ { @@ -1398,6 +1398,11 @@ "id": "workspace-generators", "file": "shared/deprecated/workspace-generators" }, + { + "name": "Custom Task Runners", + "id": "custom-task-runners", + "file": "shared/deprecated/custom-task-runners" + }, { "name": "Workspace Executors", "id": "workspace-executors", diff --git a/docs/shared/deprecated/custom-task-runners.md b/docs/shared/deprecated/custom-task-runners.md new file mode 100644 index 0000000000000..293a0d7148c54 --- /dev/null +++ b/docs/shared/deprecated/custom-task-runners.md @@ -0,0 +1,12 @@ +# tasksRunnerOptions and cacheDirectory + +As of Nx 20, custom task runners (defined via `tasksRunnerOptions`) and the `cacheDirectory` property in `nx.json` are deprecated. They will cease to function in Nx 21. In Nx 20, the local cache is stored in a database, rather than using the file system. This has two benefits: + +1. Cache reads and writes are faster. +2. The local cache is more secure since other processes with access to the file system can no longer read or modify the cache. + +For most organizations, this feature is a net positive. If you are currently using a custom task runner, you are mostly likely using it to define your own custom [remote cache](/ci/features/remote-cache) storage location. You have several options moving forward: + +1. Use [Nx Cloud](/nx-cloud) for your remote cache +2. Use an [Nx Powerpack]() plugin to store your remote cache on an [AWS S3 bucket](/nx-api/powerpack-s3-cache) or a [network drive]() +3. Use the deprecated custom task runner feature until Nx 21 diff --git a/docs/shared/features/powerpack/custom-caching.md b/docs/shared/features/powerpack/custom-caching.md index 9d22c3bd0ce1e..6093058478746 100644 --- a/docs/shared/features/powerpack/custom-caching.md +++ b/docs/shared/features/powerpack/custom-caching.md @@ -1 +1,33 @@ # Change the Remote Cache Storage Location + +The recommended way to enable the [remote cache]() is to use [Nx Replay]() and have Nx Cloud share the task cache across your organization. For those organizations that are unable to use Nx Cloud, Nx offers official plugins that are enabled by [Nx Powerpack]() to customize your remote cache location in a fast and secure manner. + +In Nx 20, the local cache mechanism is getting faster and more secure by storing the cache in a database instead of directly on the file system. The new cache is faster because it avoids the unavoidably slow speed of interacting with the file system. The new cache is more secure because it eliminates the possibility that other processes with file system access could read or modify the cache. The [`@nx/powerpack-s3-cache`](/nx-api/powerpack-s3-cache) and [`@nx/powerpack-shared-fs-cache`]() plugins enable you to leverage that speed and security with a custom remote cache storage location. + +{% callout type="note" title="Custom Task Runners are Deprecated in Nx 20" %} +If you have a custom coded method for changing the remote cache location that uses either `tasksRunnerOptions` or `cacheDirectory`, you'll need to choose a migration path. Those properties are [deprecated in Nx 20](/deprecated/custom-task-runners) and will be removed in Nx 21. +{% /callout %} + +## Custom Remote Cache Storage Plugins Require Nx Powerpack + +The `@nx/powerpack-s3-cache` and `@nx/powerpack-shared-fs-cache` plugins require an Nx Powerpack license to function. [Activating Powerpack](/recipes/installation/activate-powerpack) is a simple process. + +{% call-to-action title="Buy a Powerpack License" icon="nx" description="Unlock all the features of Nx" url="https://nx.app/nx-powerpack/purchase" /%} + +## Choose Your Storage Solution + +Read the individual plugin documentation for specific set up instructions. + +{% cards cols="2" lgCols="2" mdCols="2" smCols="2" %} + +{% link-card title="AWS S3 Bucket Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-s3-cache" icon="aws" /%} + +{% link-card title="Shared Network Drive Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-s3-cache" icon="network" /%} + +{% /cards %} + +## Switch to Nx Cloud + +These custom remote cache storage solutions only provide the remote cache functionality of Nx Cloud. If you want to leverage [distributed task execution](), [re-running flaky tasks]() or [automatically splitting tasks](), you'll need to [connect to Nx Cloud]() and use Nx Cloud's remote cache solution instead. + +{% call-to-action title="Connect to Nx Cloud" icon="nxcloud" description="Enable task distribution and Atomizer" url="/ci/intro/connect-to-nx-cloud" /%} diff --git a/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md b/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md index 51c0779b647d0..80ba95ac9ac8c 100644 --- a/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md +++ b/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md @@ -1,6 +1,90 @@ --- title: Overview of the Nx powerpack-s3-cache Plugin -description: The Nx Plugin for esbuild contains executors and generators that support building applications using esbuild. This page also explains how to configure esbuild on your Nx workspace. +description: The powerpack-s3-cache Nx plugin enables you to use an AWS S3 bucket to host your remote cache instead of Nx Cloud --- -Powerpack s3-cache +{% callout title="This plugin requires an active Nx Powerpack license" %} +In order to use `@nx/powerpack-owners`, you need to have an active Powerpack license. If you don't have a license or it has expired, the syncing process will stop working and you'll need to manually maintain your CODEOWNERS file. +{% /callout %} + +## Setup + +### 1. Install the Package + +1. [Activate Powerpack](/recipes/installation/activate-powerpack) if you haven't already +2. Install the package + +```shell +nx add @nx/powerpack-owners +``` + +### 2. Authenticate with AWS + +There are four different ways to authenticate with AWS. They will be attempted in this order: + +1. Environment variables +2. INI config files +3. Single sign-on +4. `nx.json` settings + +#### Environment Variables + +[AWS provides environment variables](https://docs.aws.amazon.com/sdkref/latest/guide/environment-variables.html) that can be used to authenticate: + +| **Environment Variable** | **Description** | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `AWS_ACCESS_KEY_ID` | The access key for your AWS account. | +| `AWS_SECRET_ACCESS_KEY` | The secret key for your AWS account. | +| `AWS_SESSION_TOKEN` | The session key for your AWS account. This is only needed when you are using temporary credentials. | +| `AWS_CREDENTIAL_EXPIRATION` | The expiration time of the credentials contained in the environment variables described above. This value must be in a format compatible with the [ISO-8601 standard](https://en.wikipedia.org/wiki/ISO_8601) and is only needed when you are using temporary credentials. | + +Both the `AWS_ACCESS_KEY_ID` and the `AWS_SECRET_ACCESS_KEY` environment variables are required to use the environment variable authentication method. + +#### INI Config Files + +AWS can read your authentication credentials from [shared INI config files](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html). The files are located at `~/.aws/credentials` and `~/.aws/config`. Both files are expected to be INI formatted with section names corresponding to profiles. Sections in the credentials file are treated as profile names, whereas profile sections in the config file must have the format of `[profile profile-name]`, except for the default profile. Profiles that appear in both files will not be merged, and the version that appears in the credentials file will be given precedence over the profile found in the config file. + +#### Single Sign-On + +Nx can read the active access token [created after running `aws sso login`](https://docs.aws.amazon.com/sdkref/latest/guide/understanding-sso.html) then request temporary AWS credentials. You can create the `AwsCredentialIdentityProvider` functions using the inline SSO parameters (`ssoStartUrl`, `ssoAccountId`, `ssoRegion`, `ssoRoleName`) or load them from [AWS SDKs and Tools shared configuration and credentials files](https://docs.aws.amazon.com/credref/latest/refdocs/creds-config-files.html). Profiles in the `credentials` file are given precedence over profiles in the `config` file. + +#### Credentials in `nx.json` File + +Storing your credentials in the `nx.json` file is the least secure of the 4 authentication options, since anyone with read access to your code base will have access to your AWS credentials. + +```jsonc {% fileName="nx.json" %} +{ + "s3": { + "ssoProfile": "default", + "accessKeyId": "MYACCESSKEYID", + "secretAccessKey": "MYSECRETACCESSKEY" + } +} +``` + +| **Property** | **Description** | +| ------------------- | ----------------------------------------------------------------------------- | +| **ssoProfile** | The name of the profile to use from your AWS CLI SSO Configuration (optional) | +| **endpoint** | The AWS endpoint URL (optional) | +| **accessKeyId** | AWS Access Key ID (optional) | +| **secretAccessKey** | AWS secret access key (optional) | + +### 3. Configure S3 Cache + +Regardless of how you manage your AWS authentication, you need to configure your Nx cache in the `nx.json` file. The `bucket` that you specify needs to already exist - Nx doesn't create it for you. + +```jsonc {% fileName="nx.json" %} +{ + "s3": { + "region": "us-east-1", + "bucket": "my-bucket", + "encryptionKey": "create-your-own-key" + } +} +``` + +| **Property** | **Description** | +| ----------------- | --------------------------------------------------------------------------------- | +| **region** | The id of the AWS region to use | +| **bucket** | The name of the AWS bucket to use | +| **encryptionKey** | Nx encryption key used to encrypt and decrypt artifacts from the cache (optional) | diff --git a/docs/shared/recipes/installation/activate-powerpack.md b/docs/shared/recipes/installation/activate-powerpack.md index 8547a525d543e..a0586ba015297 100644 --- a/docs/shared/recipes/installation/activate-powerpack.md +++ b/docs/shared/recipes/installation/activate-powerpack.md @@ -2,9 +2,9 @@ Nx Powerpack unlocks features of Nx that are particularly useful for larger organizations. The features include the ability to: -- [Run language-agnostic conformance rules](/features/powerpack-features/conformance) -- [Define code ownership at the project level](/features/powerpack-features/owners) -- [Change the remote cache storage location](/features/powerpack-features/custom-caching) +- [Run language-agnostic conformance rules](/features/powerpack/conformance) +- [Define code ownership at the project level](/features/powerpack/owners) +- [Change the remote cache storage location](/features/powerpack/custom-caching) Activating Powerpack is a two step process. diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index feeee3e0fce3f..96b6d5041c426 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -21,10 +21,10 @@ - [Automate Updating Dependencies](/features/automate-updating-dependencies) - [Enforce Module Boundaries](/features/enforce-module-boundaries) - [Manage Releases](/features/manage-releases) - - [Powerpack Features](/features/powerpack-features) - - [Run Language-Agnostic Conformance Rules](/features/powerpack-features/conformance) - - [Define Code Ownership at the Project Level](/features/powerpack-features/owners) - - [Change the Remote Cache Storage Location](/features/powerpack-features/custom-caching) + - [Powerpack Features](/features/powerpack) + - [Run Language-Agnostic Conformance Rules](/features/powerpack/conformance) + - [Define Code Ownership at the Project Level](/features/powerpack/owners) + - [Change the Remote Cache Storage Location](/features/powerpack/custom-caching) - [CI Features](/features/ci-features) - [Concepts](/concepts) - [Mental Model](/concepts/mental-model) @@ -224,6 +224,7 @@ - [workspace.json](/deprecated/workspace-json) - [As Provided vs. Derived](/deprecated/as-provided-vs-derived) - [Workspace Generators](/deprecated/workspace-generators) + - [Custom Task Runners](/deprecated/custom-task-runners) - [Workspace Executors](/deprecated/workspace-executors) - [runtimeCacheInputs](/deprecated/runtime-cache-inputs) - [cacheableOperations](/deprecated/cacheable-operations)