From a7b3c5a2c00b9382b9debef522f483e669179db1 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Thu, 20 Feb 2025 14:56:20 -0500 Subject: [PATCH 1/2] Drop node manifests --- .circleci/config.yml | 8 - docs/docs/conceptual/content-sync.md | 12 - .../reference/release-notes/v3.7/index.md | 20 - .../creating-a-source-plugin/part-7/index.mdx | 1 - .../__tests__/create-node-manifest.test.js | 242 ------- .../node-manifest/gatsby-node.js | 219 ------- integration-tests/node-manifest/package.json | 24 - .../src/pages/connection-list-query-page.js | 23 - .../src/pages/static-query-list-query.js | 24 - .../src/pages/{TestFsRouteType.id}.js | 18 - .../node-manifest/src/templates/four.js | 17 - .../node-manifest/src/templates/one.js | 18 - .../node-manifest/src/templates/three.js | 17 - .../node-manifest/src/templates/two.js | 14 - .../node-manifest/utils/get-gatsby-process.js | 17 - .../src/structured-errors/error-map.ts | 51 -- .../src/__tests__/normalize.js | 5 - .../gatsby-source-contentful/src/normalize.js | 80 --- .../src/source-nodes.js | 2 - .../gatsby-source-drupal/src/gatsby-node.ts | 14 +- packages/gatsby-source-drupal/src/utils.ts | 65 +- .../src/steps/preview/index.ts | 29 - packages/gatsby/index.d.ts | 11 - packages/gatsby/src/query/query-watcher.ts | 8 - packages/gatsby/src/redux/actions/internal.ts | 7 - packages/gatsby/src/redux/actions/public.js | 34 +- packages/gatsby/src/redux/reducers/index.ts | 2 - .../src/redux/reducers/node-manifest.ts | 80 --- packages/gatsby/src/redux/types.ts | 23 - .../src/utils/__tests__/node-manifest.ts | 606 ------------------ packages/gatsby/src/utils/node-manifest.ts | 513 --------------- packages/gatsby/src/utils/page-data.ts | 37 -- 32 files changed, 3 insertions(+), 2238 deletions(-) delete mode 100644 integration-tests/node-manifest/__tests__/create-node-manifest.test.js delete mode 100644 integration-tests/node-manifest/gatsby-node.js delete mode 100644 integration-tests/node-manifest/package.json delete mode 100644 integration-tests/node-manifest/src/pages/connection-list-query-page.js delete mode 100644 integration-tests/node-manifest/src/pages/static-query-list-query.js delete mode 100644 integration-tests/node-manifest/src/pages/{TestFsRouteType.id}.js delete mode 100644 integration-tests/node-manifest/src/templates/four.js delete mode 100644 integration-tests/node-manifest/src/templates/one.js delete mode 100644 integration-tests/node-manifest/src/templates/three.js delete mode 100644 integration-tests/node-manifest/src/templates/two.js delete mode 100644 integration-tests/node-manifest/utils/get-gatsby-process.js delete mode 100644 packages/gatsby/src/redux/reducers/node-manifest.ts delete mode 100644 packages/gatsby/src/utils/__tests__/node-manifest.ts delete mode 100644 packages/gatsby/src/utils/node-manifest.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index a460d2db8e752..e06bf4b4466da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,12 +238,6 @@ jobs: - e2e-test: test_path: integration-tests/gatsby-source-wordpress - integration_tests_node_manifest: - executor: node - steps: - - e2e-test: - test_path: integration-tests/node-manifest - integration_tests_long_term_caching: executor: node steps: @@ -681,8 +675,6 @@ workflows: - bootstrap - integration_tests_gatsby_source_wordpress: <<: *e2e-test-workflow - - integration_tests_node_manifest: - <<: *e2e-test-workflow - integration_tests_long_term_caching: <<: *e2e-test-workflow - integration_tests_cache_resilience: diff --git a/docs/docs/conceptual/content-sync.md b/docs/docs/conceptual/content-sync.md index 57d3743c9566b..833811412713d 100644 --- a/docs/docs/conceptual/content-sync.md +++ b/docs/docs/conceptual/content-sync.md @@ -26,17 +26,6 @@ In the case that your content lives on multiple pages, for example a blog post p You will not need to do this if you're building pages using the [File System Route API][fsroutesapi], or if your page context includes a matching `id` property. See the next section for more info on this. -## Node to Page Mapping Hierarchy - -Content Sync uses the [`unstable_createNodeManifest`][createnodemanifest] API via source plugins to allow source plugins to tell Gatsby which nodes are being previewed. When this public action is called, Gatsby uses an internal hierarchy to determine which page the content author intends to preview. - -The hierarchy is as follows, from most specific to least specific: - -1. [The`ownerNodeId` property in the `createPage` action][createpage]. (This is set manually by the Gatsby site developer) -2. Nodes associated with pages created from the [File System Route API][fsroutesapi]. (automatic) -3. An `id` property in the [page `context` passed to the `createPage` API][createpage] with a node id which matches the previewed node id. (automatic) -4. The first matching node id found in Gatsby's [GraphQL query tracking][querytracking] which maps node id's to pages that query them. This allows nodes which have no direct top-level page correlated with them to be previewed throughout the site. (automatic) - ## Source Plugin Authors If you're a source plugin author, you can find instructions on adding Content Sync support to your source plugin and CMS extension in the [source plugin author docs](https://www.gatsbyjs.com/docs/how-to/plugins-and-themes/creating-a-source-plugin/). @@ -46,7 +35,6 @@ If you're a source plugin author, you can find instructions on adding Content Sy ![Diagram of Content Sync on Gatsby Cloud. When a user clicks "Open Preview" the manifest id is created from the CMS and sent to Gatsby Cloud. The Content Sync UI then displays a loading state while it checks for a matching manifest id in Gatsby build public directory. At the same time a request is send to the preview webhook and Gatsby Cloud starts a build. If it finds a matching manifest id created from the CMS Gatsby maps data to correct page for preview and adds the manifest file to the build public directory. Now the Gatsby Cloud Preview Runner detects a node manifest to preview and redirects to the preview itself.](../images/content-sync-diagram.png) [authordocs]: /docs/how-to/plugins-and-themes/creating-a-source-plugin/#enabling-content-sync -[createnodemanifest]: /docs/reference/config-files/actions#unstable_createNodeManifest [createpage]: /docs/reference/config-files/actions#createPage [querytracking]: /docs/page-node-dependencies/ [fsroutesapi]: /docs/reference/routing/file-system-route-api/ diff --git a/docs/docs/reference/release-notes/v3.7/index.md b/docs/docs/reference/release-notes/v3.7/index.md index 95eb4b9e7b9c6..b3e2701a71e0d 100644 --- a/docs/docs/reference/release-notes/v3.7/index.md +++ b/docs/docs/reference/release-notes/v3.7/index.md @@ -11,7 +11,6 @@ Key highlights of this release: - [Functions](#functions) - Now generally available - [webpack caching](#webpack-caching) - Starting gradual rollout - [Yarn 2 (PNP) support](#yarn-2-pnp-support) -- [New API for source plugins: `createNodeManifest`](#new-api-for-source-plugins-createnodemanifest) - [Experimental: Node persistence in LMDB](#experimental-node-persistence-in-lmdb) - Lower peak memory usage - [`gatsby-remark-images`: async image decoding by default](#gatsby-remark-images-async-image-decoding-by-default) @@ -42,25 +41,6 @@ behind a flag. Now we are starting a gradual rollout for everyone. This release If you encounter any issues, please let us know in the [umbrella discussion](https://github.com/gatsbyjs/gatsby/discussions/31525). -## New API for source plugins: `createNodeManifest` - -This new API will be used to write out information that ties the state of some CMS content to a finally built page. -Useful for routing to previews or even production content once they're built. - -This release adds a new public action `unstable_createNodeManifest` which is used to tie a manifest ID -(a CMS id that maps to a unique revision state of some content) to a page that was built from a specific node. - -To make the mapping from node to page more accurate, this release introduces a new argument to the `createPage` helper, -`ownerNodeId` so that a user can specify that a page is owned by a specific node. - -In the case that no `ownerNodeId` is provided, the logic checks for a page with an `id` variable in page context that -matches to the node id of the node manifest. If neither of those exist the logic maps the manifest to the first -page the node is found on in query tracking. - -The result is that a source plugin can allow a CMS to create a manifest file using a CMS ID that maps to a finally built page in Gatsby, allowing for a service to redirect to the right page after a build is complete. - -[Original PR](https://github.com/gatsbyjs/gatsby/pull/31127). - ## Experimental: Node persistence in LMDB This release introduces a new experimental data storage option: LMDB (via the excellent [lmdb-store](https://github.com/DoctorEvidence/lmdb-store) package). diff --git a/docs/docs/tutorial/creating-a-source-plugin/part-7/index.mdx b/docs/docs/tutorial/creating-a-source-plugin/part-7/index.mdx index c5fccd2752747..98c00583685d7 100644 --- a/docs/docs/tutorial/creating-a-source-plugin/part-7/index.mdx +++ b/docs/docs/tutorial/creating-a-source-plugin/part-7/index.mdx @@ -176,7 +176,6 @@ Here are some things to keep in mind and some "gotchas" depending on how the CMS - Inside the CMS, sometimes you will need to wait to make sure you have the correct `updatedAt` timestamp as some CMS may take a second to update their backend and then wait for the change to propagate to the frontend. While others will immediately update the frontend and then propagate that to the backend. You will need the _most_ up to date timestamp when opening the Content Sync UI waiting room. - Make sure that a preview webhook is being sent to Gatsby Cloud after the content is edited, whether it's before you press the "Open Preview" button or the "Open Preview" is the trigger that sends the webhook. -- The node manifests get written out in the `public` dir of your gatsby site, so you can check the manifests on your local disk at `/public/__node-manifests//.json` or you can navigate directly to that piece of content `https:///__node-manifests//`. ## MIME types diff --git a/integration-tests/node-manifest/__tests__/create-node-manifest.test.js b/integration-tests/node-manifest/__tests__/create-node-manifest.test.js deleted file mode 100644 index 3c8cf2f176cb9..0000000000000 --- a/integration-tests/node-manifest/__tests__/create-node-manifest.test.js +++ /dev/null @@ -1,242 +0,0 @@ -// Must run `gatsby-dev --packages gatsby to get latest gatsby changes before running the tests -jest.setTimeout(100000) -const DEFAULT_MAX_DAYS_OLD = 30 - -const { - spawnGatsbyProcess, - runGatsbyClean, -} = require(`../utils/get-gatsby-process`) -const urling = require(`urling`) -const rimraf = require(`rimraf`) -const path = require(`path`) -const fs = require(`fs-extra`) - -const gatsbyCommandName = process.env.GATSBY_COMMAND_NAME || `develop` - -const manifestDir = path.join( - process.cwd(), - `public`, - `__node-manifests`, - `default-site-plugin` -) - -const pageDataDir = path.join(process.cwd(), `public`, `page-data`) - -const createManifestId = nodeId => `${gatsbyCommandName}-${nodeId}` - -const cleanNodeManifests = async () => { - console.log(`removing any lingering node manifest files at ${manifestDir}`) - return new Promise(resolve => rimraf(manifestDir, resolve)) -} - -const getManifestContents = async nodeId => - await fs.readJSON(path.join(manifestDir, `${createManifestId(nodeId)}.json`)) - -const pageDataContents = async pagePath => - await fs.readJSON(path.join(pageDataDir, pagePath, `page-data.json`)) - -const port = 8010 - -// see gatsby-node.js for where createNodeManifest was called -// and for the corresponding pages that were created with createPage -describe(`Node Manifest API in "gatsby ${gatsbyCommandName}"`, () => { - let gatsbyProcess - - beforeAll(async () => { - await cleanNodeManifests() - - gatsbyProcess = spawnGatsbyProcess(gatsbyCommandName, { - PORT: port, - }) - - if (gatsbyCommandName === `develop`) { - // wait for localhost - return urling(`http://localhost:${port}`) - } else if (gatsbyCommandName === `build`) { - // for gatsby build wait for the process to exit - return gatsbyProcess - } - }) - - afterAll(() => { - return new Promise(resolve => { - if ( - !gatsbyProcess || - gatsbyProcess.killed || - gatsbyProcess.exitCode !== null - ) { - return resolve() - } - - gatsbyProcess.on(`exit`, () => { - setImmediate(() => { - resolve() - }) - }) - - gatsbyProcess.kill() - }) - }) - - it(`Creates an accurate node manifest when using the ownerNodeId argument in createPage`, async () => { - const manifestFileContents = await getManifestContents(1) - - expect(manifestFileContents.node.id).toBe(`1`) - expect(manifestFileContents.page.path).toBe(`/one/`) - expect(manifestFileContents.foundPageBy).toBe(`ownerNodeId`) - }) - - it(`Creates an accurate node manifest when ownerNodeId isn't present but there's a matching "id" in pageContext`, async () => { - const manifestFileContents = await getManifestContents(2) - - expect(manifestFileContents.node.id).toBe(`2`) - expect(manifestFileContents.page.path).toBe(`/two/`) - expect(manifestFileContents.foundPageBy).toBe(`context.id`) - }) - - it(`Creates an accurate node manifest when ownerNodeId isn't present but there's a matching "slug" in pageContext`, async () => { - const manifestFileContents = await getManifestContents(5) - - expect(manifestFileContents.node.id).toBe(`5`) - expect(manifestFileContents.page.path).toBe(`/slug-test-path/`) - expect(manifestFileContents.foundPageBy).toBe(`context.slug`) - }) - - if (gatsbyCommandName === `build`) { - // this doesn't work in gatsby develop since query tracking - // only runs when visiting a page in browser. - it(`Creates a node manifest from the first tracked page it finds in query tracking`, async () => { - const manifestFileContents = await getManifestContents(3) - - expect(manifestFileContents.node.id).toBe(`3`) - expect( - [`/three/`, `/three-alternative/`].includes( - manifestFileContents.page.path - ) - ).toBe(true) - expect(manifestFileContents.foundPageBy).toBe(`queryTracking`) - }) - - // this doesn't work in gatsby develop since page-data.json files aren't written out - it(`Adds the correct manifestId to the pageData.json digest`, async () => { - const nodeId = `1` - const path = `one` - const pageData = await pageDataContents(path) - - expect(pageData.manifestId).toEqual(createManifestId(nodeId)) - }) - } - - it(`Creates a node manifest with a null page path when createNodeManifest is called but a page is not created for the provided node in the Gatsby site`, async () => { - const manifestFileContents = await getManifestContents(4) - - expect(manifestFileContents.node.id).toBe(`4`) - expect(manifestFileContents.page.path).toBe(null) - expect(manifestFileContents.foundPageBy).toBe(`none`) - }) - - it(`Creates a Node manifest for filesystem routes`, async () => { - const manifestFileContents = await getManifestContents(`filesystem-1`) - - expect(manifestFileContents.node.id).toBe(`filesystem-1`) - expect(manifestFileContents.page.path).toBe(`/filesystem-1/`) - expect(manifestFileContents.foundPageBy).toBe(`filesystem-route-api`) - }) - - it(`Creates a manifest for the node when updatedAt is included and is within ${DEFAULT_MAX_DAYS_OLD} days`, async () => { - const recentlyUpdatedNodeId = `updatedAt-1` - const staleNodeId = `updatedAt-2` - const recentlyUpdatedNodeManifest = await getManifestContents( - recentlyUpdatedNodeId - ) - - try { - await getManifestContents(staleNodeId) - throw new Error() - } catch (e) { - expect(e.message).toContain(`no such file or directory`) - expect(e.message).toContain(createManifestId(staleNodeId)) - } - - expect(recentlyUpdatedNodeManifest.node.id).toBe(recentlyUpdatedNodeId) - }) - - it(`Creates a correct node manifest for nodes in connection list queries`, async () => { - const manifestFileContents1 = await getManifestContents( - `connection-list-query-node` - ) - - expect(manifestFileContents1.node.id).toBe(`connection-list-query-node`) - expect(manifestFileContents1.page.path).toBe(`/connection-list-query-page/`) - - const manifestFileContents2 = await getManifestContents( - `connection-list-query-node-2` - ) - - expect(manifestFileContents2.node.id).toBe(`connection-list-query-node-2`) - expect(manifestFileContents2.page.path).toBe(`/connection-list-query-page/`) - }) - - it(`Creates a correct node manifest for nodes in connection list queries using staticQuery()`, async () => { - const manifestFileContents1 = await getManifestContents( - `static-query-list-query-node` - ) - - expect(manifestFileContents1.node.id).toBe(`static-query-list-query-node`) - expect(manifestFileContents1.page.path).toBe(`/static-query-list-query/`) - - const manifestFileContents2 = await getManifestContents( - `static-query-list-query-node-2` - ) - - expect(manifestFileContents2.node.id).toBe(`static-query-list-query-node-2`) - expect(manifestFileContents2.page.path).toBe(`/static-query-list-query/`) - }) -}) - -describe(`Node Manifest API in "gatsby ${gatsbyCommandName}"`, () => { - let gatsbyProcess - - beforeEach(async () => { - await runGatsbyClean() - - gatsbyProcess = spawnGatsbyProcess(gatsbyCommandName, { - DUMMY_NODE_MANIFEST_COUNT: 700, - NODE_MANIFEST_FILE_LIMIT: 500, - PORT: port, - }) - - if (gatsbyCommandName === `develop`) { - // wait for localhost - await urling(`http://localhost:${port}`) - } else if (gatsbyCommandName === `build`) { - // for gatsby build wait for the process to exit - return gatsbyProcess - } - }) - - afterAll(() => { - return new Promise(resolve => { - if ( - !gatsbyProcess || - gatsbyProcess.killed || - gatsbyProcess.exitCode !== null - ) { - return resolve() - } - - gatsbyProcess.on(`exit`, () => { - setImmediate(() => { - resolve() - }) - }) - - gatsbyProcess.kill() - }) - }) - - it(`Limits the number of node manifest files written to disk to 500`, async () => { - const nodeManifestFiles = fs.readdirSync(manifestDir) - expect(nodeManifestFiles).toHaveLength(500) - }) -}) diff --git a/integration-tests/node-manifest/gatsby-node.js b/integration-tests/node-manifest/gatsby-node.js deleted file mode 100644 index ecb11eb036378..0000000000000 --- a/integration-tests/node-manifest/gatsby-node.js +++ /dev/null @@ -1,219 +0,0 @@ -const commandName = process.env.NODE_ENV === `development` ? `develop` : `build` -const DEFAULT_MAX_DAYS_OLD = 30 -const createManifestId = nodeId => `${commandName}-${nodeId}` - -const DUMMY_NODE_MANIFEST_COUNT = process.env.DUMMY_NODE_MANIFEST_COUNT || 0 -const ONE_DAY = 1000 * 60 * 60 * 24 -const THIRTY_DAYS = ONE_DAY * 30 -const YESTERDAY = new Date() - ONE_DAY -const LOWER_CREATED_AT_TIME_LIMIT = YESTERDAY - THIRTY_DAYS -const UPPER_CREATED_AT_TIME_LIMIT = YESTERDAY - -function randomIntFromInterval(min, max) { - return Math.floor(Math.random() * (max - min + 1) + min) -} - -exports.sourceNodes = ({ actions }) => { - // template nodes - for (let id = 1; id < 6; id++) { - const node = { - id: `${id}`, - internal: { - type: `TestNode`, - contentDigest: `${id}`, - }, - } - - if (id === 5) { - node.slug = `test-slug` - } - - actions.createNode(node) - - actions.unstable_createNodeManifest({ - manifestId: createManifestId(id), - node, - }) - } - - // These nodes are intended to be added when running tests for the node manifest creation limit logic - for (let i = 0; i < DUMMY_NODE_MANIFEST_COUNT; i++) { - const id = `dummy-${i}` - const node = { - id, - internal: { - type: `TestNode`, - contentDigest: id, - }, - } - - const updatedAtUTC = new Date( - randomIntFromInterval( - LOWER_CREATED_AT_TIME_LIMIT, - UPPER_CREATED_AT_TIME_LIMIT - ) - ).toUTCString() - - actions.createNode(node) - - actions.unstable_createNodeManifest({ - manifestId: createManifestId(id), - node, - updatedAtUTC, - }) - } - - // filesystem route api node - const node = { - id: `filesystem-1`, - internal: { - type: `TestFSRouteType`, - contentDigest: `1`, - }, - } - - actions.createNode(node) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(node.id), - node, - }) - const today = new Date() - const nodeTooOldToGetManifestCreated = new Date( - new Date().setDate(today.getDate() - (DEFAULT_MAX_DAYS_OLD + 1)) - ).toISOString() - - const nodeUpdated1 = { - id: `updatedAt-1`, - internal: { - type: `TestUpdatedAtType`, - contentDigest: `1`, - }, - updatedAt: today.toISOString(), - } - - const nodeUpdated2 = { - id: `updatedAt-2`, - internal: { - type: `TestUpdatedAtType`, - contentDigest: `1`, - }, - updatedAt: nodeTooOldToGetManifestCreated, - } - - actions.createNode(nodeUpdated1) - actions.createNode(nodeUpdated2) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeUpdated1.id), - node: nodeUpdated1, - updatedAtUTC: nodeUpdated1.updatedAt, - }) - - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeUpdated2.id), - node: nodeUpdated2, - updatedAtUTC: nodeUpdated2.updatedAt, - }) - - const nodeForConnectionListQuery1 = { - id: `connection-list-query-node`, - title: `First connection list query node`, - internal: { - type: `TestConnectionListQueryType`, - contentDigest: `1`, - }, - } - const nodeForConnectionListQuery2 = { - id: `connection-list-query-node-2`, - title: `Second connection list query node`, - internal: { - type: `TestConnectionListQueryType`, - contentDigest: `2`, - }, - } - - actions.createNode(nodeForConnectionListQuery1) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeForConnectionListQuery1.id), - node: nodeForConnectionListQuery1, - }) - - actions.createNode(nodeForConnectionListQuery2) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeForConnectionListQuery2.id), - node: nodeForConnectionListQuery2, - }) - - const nodeForStaticQueryList1 = { - id: `static-query-list-query-node`, - title: `First static query list query node`, - internal: { - type: `TestConnectionStaticQueryListQueryType`, - contentDigest: `1`, - }, - } - const nodeForStaticQueryList2 = { - id: `static-query-list-query-node-2`, - title: `Second static query list query node`, - internal: { - type: `TestConnectionStaticQueryListQueryType`, - contentDigest: `2`, - }, - } - - actions.createNode(nodeForStaticQueryList1) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeForStaticQueryList1.id), - node: nodeForStaticQueryList1, - }) - - actions.createNode(nodeForStaticQueryList2) - actions.unstable_createNodeManifest({ - manifestId: createManifestId(nodeForStaticQueryList2.id), - node: nodeForStaticQueryList2, - }) -} - -/** - * The main way node manifests are mapped to pages is via the ownerNodeId argument to createPage. - * If it doesn't exist we either grab the first page with `id` in pageContext that matches to our nodeManifest's node id, or if that doesn't exist, the first page query where our nodeManifest's node id is queried. Node 2 and 3 are used to test this behaviour (in that order). - * - * Note that there's no page created for our node with an id of `4`. The Node Manifest API should account for the situation where the user didn't create a page for a node that a manifest was created for by a source plugin. - */ -exports.createPages = ({ actions }) => { - actions.createPage({ - path: `one`, - ownerNodeId: `1`, - component: require.resolve(`./src/templates/one.js`), - }) - - actions.createPage({ - path: `two`, - component: require.resolve(`./src/templates/two.js`), - context: { - id: `2`, - }, - }) - - actions.createPage({ - path: `two-alternative`, - component: require.resolve(`./src/templates/two.js`), - }) - - actions.createPage({ - path: `three`, - component: require.resolve(`./src/templates/three.js`), - }) - - actions.createPage({ - path: `three-alternative`, - component: require.resolve(`./src/templates/three.js`), - }) - - actions.createPage({ - path: `slug-test-path`, - context: { - slug: `test-slug`, - }, - component: require.resolve(`./src/templates/four.js`), - }) -} diff --git a/integration-tests/node-manifest/package.json b/integration-tests/node-manifest/package.json deleted file mode 100644 index b0893e5a50e9c..0000000000000 --- a/integration-tests/node-manifest/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "gatsby-node-manifest-test-site", - "version": "0.1.0", - "description": "A test site for Gatsby's node manifest API", - "main": "index.js", - "scripts": { - "test": "cross-env GATSBY_COMMAND_NAME=build jest --runInBand && cross-env GATSBY_COMMAND_NAME=develop jest --runInBand" - }, - "author": "Tyler Barnes ", - "license": "MIT", - "dependencies": { - "gatsby": "next", - "react": "^18.2.0", - "react-dom": "^18.2.0" - }, - "devDependencies": { - "cross-env": "^7.0.3", - "execa": "^5.1.1", - "fs-extra": "^10.0.0", - "jest": "^29.3.1", - "rimraf": "^3.0.2", - "urling": "^1.0.7" - } -} diff --git a/integration-tests/node-manifest/src/pages/connection-list-query-page.js b/integration-tests/node-manifest/src/pages/connection-list-query-page.js deleted file mode 100644 index 6b671e933f3a4..0000000000000 --- a/integration-tests/node-manifest/src/pages/connection-list-query-page.js +++ /dev/null @@ -1,23 +0,0 @@ -import React from "react" -import { graphql } from "gatsby" - -export default function ConnectionListQueryPage({ data }) { - return ( -
-

ConnectionListQueryPage

- {data.allTestConnectionListQueryType.nodes.map(({ title }) => ( -

{title}

- ))} -
- ) -} - -export const query = graphql` - query ConnectionListQueryPage { - allTestConnectionListQueryType { - nodes { - title - } - } - } -` diff --git a/integration-tests/node-manifest/src/pages/static-query-list-query.js b/integration-tests/node-manifest/src/pages/static-query-list-query.js deleted file mode 100644 index 19b9fa86babdb..0000000000000 --- a/integration-tests/node-manifest/src/pages/static-query-list-query.js +++ /dev/null @@ -1,24 +0,0 @@ -import React from "react" -import { graphql } from "gatsby" -import { useStaticQuery } from "gatsby" - -export default function TestConnectionStaticQueryListQueryType() { - const data = useStaticQuery(graphql` - query TestConnectionStaticQueryListQueryType { - allTestConnectionStaticQueryListQueryType { - nodes { - title - } - } - } - `) - - return ( -
-

TestConnectionStaticQueryListQueryType

- {data.allTestConnectionStaticQueryListQueryType.nodes.map(({ title }) => ( -

{title}

- ))} -
- ) -} diff --git a/integration-tests/node-manifest/src/pages/{TestFsRouteType.id}.js b/integration-tests/node-manifest/src/pages/{TestFsRouteType.id}.js deleted file mode 100644 index 6df49dcbc7383..0000000000000 --- a/integration-tests/node-manifest/src/pages/{TestFsRouteType.id}.js +++ /dev/null @@ -1,18 +0,0 @@ -import { graphql } from "gatsby" -import React from "react" - -export default function FSRouteTest({ data }) { - return
filesystem route. id {data.testFsRouteType.id}
-} - -export const query = graphql` - query FSRouteTest($id: String!) { - testFsRouteType(id: { eq: $id }) { - id - } - # querying this other node to make sure inferring the owner from context.id prevents this node from being the page owner - otherNode: testNode(id: { eq: "2" }) { - id - } - } -` diff --git a/integration-tests/node-manifest/src/templates/four.js b/integration-tests/node-manifest/src/templates/four.js deleted file mode 100644 index 1874dfccbc20f..0000000000000 --- a/integration-tests/node-manifest/src/templates/four.js +++ /dev/null @@ -1,17 +0,0 @@ -import { graphql } from "gatsby" -import React from "react" - -export default function Four({ data }) { - return
Template 4. Node by slug {data.testNode.slug}
-} - -export const query = graphql` - query SLUG_TEST($slug: String) { - testNode(slug: { eq: $slug }) { - id - } - otherNode: testNode(id: { eq: "2" }) { - id - } - } -` diff --git a/integration-tests/node-manifest/src/templates/one.js b/integration-tests/node-manifest/src/templates/one.js deleted file mode 100644 index 9abcb0a10daa8..0000000000000 --- a/integration-tests/node-manifest/src/templates/one.js +++ /dev/null @@ -1,18 +0,0 @@ -import { graphql } from "gatsby" -import React from "react" - -export default function One({ data }) { - return
Template 1. Node id {data.testNode.id}
-} - -export const query = graphql` - query { - testNode(id: { eq: "1" }) { - id - } - # querying this other node to make sure ownerNodeId in gatsby-node prevents this node from being the page owner - otherNode: testNode(id: { eq: "2" }) { - id - } - } -` diff --git a/integration-tests/node-manifest/src/templates/three.js b/integration-tests/node-manifest/src/templates/three.js deleted file mode 100644 index b0e7a8db42671..0000000000000 --- a/integration-tests/node-manifest/src/templates/three.js +++ /dev/null @@ -1,17 +0,0 @@ -import { graphql } from "gatsby" -import React from "react" - -export default function One({ data }) { - return
Template 3. Node id {data.testNode.id}
-} - -export const query = graphql` - query { - testNode(id: { eq: "3" }) { - id - } - otherNode: testNode(id: { eq: "2" }) { - id - } - } -` diff --git a/integration-tests/node-manifest/src/templates/two.js b/integration-tests/node-manifest/src/templates/two.js deleted file mode 100644 index 28deb3899e130..0000000000000 --- a/integration-tests/node-manifest/src/templates/two.js +++ /dev/null @@ -1,14 +0,0 @@ -import { graphql } from "gatsby" -import React from "react" - -export default function One({ data }) { - return
Template 2. Node id {data.testNode.id}
-} - -export const query = graphql` - query { - testNode(id: { eq: "2" }) { - id - } - } -` diff --git a/integration-tests/node-manifest/utils/get-gatsby-process.js b/integration-tests/node-manifest/utils/get-gatsby-process.js deleted file mode 100644 index 1a5ebd3852ef9..0000000000000 --- a/integration-tests/node-manifest/utils/get-gatsby-process.js +++ /dev/null @@ -1,17 +0,0 @@ -const execa = require(`execa`) -const path = require(`path`) - -function spawnGatsbyProcess(command = `develop`, env = {}) { - return execa(process.execPath, [`./node_modules/gatsby/cli.js`, command], { - stdio: [`inherit`, `inherit`, `inherit`], - env: { - ...process.env, - NODE_ENV: command === `develop` ? `development` : `production`, - ...env, - }, - }) -} - -exports.spawnGatsbyProcess = spawnGatsbyProcess - -exports.runGatsbyClean = () => spawnGatsbyProcess("clean") diff --git a/packages/gatsby-cli/src/structured-errors/error-map.ts b/packages/gatsby-cli/src/structured-errors/error-map.ts index c0aea02ca5318..0cb53e8622b6b 100644 --- a/packages/gatsby-cli/src/structured-errors/error-map.ts +++ b/packages/gatsby-cli/src/structured-errors/error-map.ts @@ -13,13 +13,6 @@ const optionalGraphQLInfo = (context: IOptionalGraphQLInfoContext): string => context.plugin ? `\nPlugin: ${context.plugin}` : `` }` -const getSharedNodeManifestWarning = (inputManifest: { - manifestId: string - node: { id: string } - pluginName: string -}): string => - `Plugin ${inputManifest.pluginName} called unstable_createNodeManifest() for node id "${inputManifest.node.id}" with a manifest id of "${inputManifest.manifestId}"` - const errors: Record = { "": { text: (context): string => { @@ -817,50 +810,6 @@ const errors: Record = { type: Type.UNKNOWN, docsUrl: `https://support.gatsbyjs.com/hc/en-us/articles/360056811354`, }, - // Node Manifest warnings - "11801": { - text: ({ inputManifest }): string => `${getSharedNodeManifestWarning( - inputManifest - )} but Gatsby couldn't find a page for this node. - If you want a manifest to be created for this node (for previews or other purposes), ensure that a page was created (and that a ownerNodeId is added to createPage() if you're not using the Filesystem Route API). See https://www.gatsbyjs.com/docs/conceptual/content-sync for more info.\n`, - level: Level.WARNING, - category: ErrorCategory.USER, - type: Type.API_NODE_VALIDATION, - }, - "11802": { - text: ({ inputManifest, pagePath }): string => - `${getSharedNodeManifestWarning( - inputManifest - )} but Gatsby didn't find an ownerNodeId for the page at ${pagePath}\nUsing the first page that was found with the node manifest id set in pageContext.id in createPage().\nThis may result in an inaccurate node manifest (for previews or other purposes). See https://www.gatsbyjs.com/docs/conceptual/content-sync for more info.`, - level: Level.WARNING, - category: ErrorCategory.USER, - type: Type.API_NODE_VALIDATION, - }, - "11805": { - text: ({ inputManifest, pagePath }): string => - `${getSharedNodeManifestWarning( - inputManifest - )} but Gatsby didn't find an ownerNodeId for the page at ${pagePath}\nUsing the first page that was found with the node manifest id set in pageContext.slug in createPage().\nThis may result in an inaccurate node manifest (for previews or other purposes). See https://www.gatsbyjs.com/docs/conceptual/content-sync for more info.`, - level: Level.WARNING, - category: ErrorCategory.USER, - type: Type.API_NODE_VALIDATION, - }, - "11803": { - text: ({ inputManifest, pagePath }): string => - `${getSharedNodeManifestWarning( - inputManifest - )} but Gatsby didn't find an ownerNodeId for the page at ${pagePath}\nUsing the first page where this node is queried.\nThis may result in an inaccurate node manifest (for previews or other purposes). See https://www.gatsbyjs.com/docs/conceptual/content-sync for more info.`, - level: Level.WARNING, - category: ErrorCategory.USER, - type: Type.API_NODE_VALIDATION, - }, - "11804": { - text: ({ pluginName, nodeId }): string => - `Plugin ${pluginName} called unstable_createNodeManifest for a node which doesn't exist with an id of ${nodeId}`, - level: Level.WARNING, - category: ErrorCategory.USER, - type: Type.API_NODE_VALIDATION, - }, // Parcel Compilation Errors "11901": { text: (context): string => diff --git a/packages/gatsby-source-contentful/src/__tests__/normalize.js b/packages/gatsby-source-contentful/src/__tests__/normalize.js index 2e1f2be440e17..ac61a7cda2bd1 100644 --- a/packages/gatsby-source-contentful/src/__tests__/normalize.js +++ b/packages/gatsby-source-contentful/src/__tests__/normalize.js @@ -32,8 +32,6 @@ const restrictedNodeFields = [ const pluginConfig = createPluginConfig({}) -const unstable_createNodeManifest = jest.fn() - // Counts the created nodes per node type function countCreatedNodeTypesFromMock(mock) { const nodeTypeCounts = {} @@ -223,7 +221,6 @@ describe(`Process contentful data (by name)`, () => { space, useNameForId: true, pluginConfig, - unstable_createNodeManifest, }) }) @@ -340,7 +337,6 @@ describe(`Process existing mutated nodes in warm build`, () => { space, useNameForId: true, pluginConfig, - unstable_createNodeManifest, }) }) @@ -450,7 +446,6 @@ describe(`Process contentful data (by id)`, () => { space, useNameForId: false, pluginConfig, - unstable_createNodeManifest, }) }) const nodeTypeCounts = countCreatedNodeTypesFromMock(createNode.mock) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index c63e61904ad6a..2fdb2af28378c 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -287,77 +287,6 @@ function prepareJSONNode(id, node, key, content) { return JSONNode } -let numberOfContentSyncDebugLogs = 0 -const maxContentSyncDebugLogTimes = 50 - -let warnOnceForNoSupport = false -let warnOnceToUpgradeGatsby = false - -/** - * This fn creates node manifests which are used for Gatsby Cloud Previews via the Content Sync API/feature. - * Content Sync routes a user from Contentful to a page created from the entry data they're interested in previewing. - */ - -function contentfulCreateNodeManifest({ - pluginConfig, - entryItem, - entryNode, - space, - unstable_createNodeManifest, -}) { - const isPreview = pluginConfig.get(`host`) === `preview.contentful.com` - - const createNodeManifestIsSupported = - typeof unstable_createNodeManifest === `function` - - const shouldCreateNodeManifest = isPreview && createNodeManifestIsSupported - - const updatedAt = entryItem.sys.updatedAt - - const manifestId = `${space.sys.id}-${entryItem.sys.id}-${updatedAt}` - - if ( - process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true` && - numberOfContentSyncDebugLogs <= maxContentSyncDebugLogTimes - ) { - numberOfContentSyncDebugLogs++ - - console.info( - JSON.stringify({ - isPreview, - createNodeManifestIsSupported, - shouldCreateNodeManifest, - manifestId, - entryItemSysUpdatedAt: updatedAt, - }) - ) - } - - if (shouldCreateNodeManifest) { - if (shouldUpgradeGatsbyVersion && !warnOnceToUpgradeGatsby) { - console.warn( - `Your site is doing more work than it needs to for Preview, upgrade to Gatsby ^${GATSBY_VERSION_MANIFEST_V2} for better performance` - ) - warnOnceToUpgradeGatsby = true - } - - unstable_createNodeManifest({ - manifestId, - node: entryNode, - updatedAtUTC: updatedAt, - }) - } else if ( - isPreview && - !createNodeManifestIsSupported && - !warnOnceForNoSupport - ) { - console.warn( - `Contentful: Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` - ) - warnOnceForNoSupport = true - } -} - function makeQueuedCreateNode({ nodeCount, createNode }) { if (nodeCount > 5000) { let createdNodeCount = 0 @@ -411,7 +340,6 @@ export const createNodesForContentType = async ({ restrictedNodeFields, conflictFieldPrefix, entries, - unstable_createNodeManifest, createNode, createNodeId, getNode, @@ -617,14 +545,6 @@ export const createNodesForContentType = async ({ }, } - contentfulCreateNodeManifest({ - pluginConfig, - entryItem, - entryNode, - space, - unstable_createNodeManifest, - }) - // Revision applies to entries, assets, and content types if (entryItem.sys.revision) { entryNode.sys.revision = entryItem.sys.revision diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index af71f76c938d6..2ecb343bdbaf4 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -66,7 +66,6 @@ export async function sourceNodes( createNode: originalCreateNode, touchNode, deleteNode: originalDeleteNode, - unstable_createNodeManifest, enableStatefulSourceNodes, } = actions @@ -526,7 +525,6 @@ export async function sourceNodes( space, useNameForId: pluginConfig.get(`useNameForId`), pluginConfig, - unstable_createNodeManifest, }) // allow node to garbage collect these items if it needs to diff --git a/packages/gatsby-source-drupal/src/gatsby-node.ts b/packages/gatsby-source-drupal/src/gatsby-node.ts index 8317b65758058..e5309653f22c6 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.ts +++ b/packages/gatsby-source-drupal/src/gatsby-node.ts @@ -29,7 +29,6 @@ import { handleWebhookUpdate, createNodeIfItDoesNotExist, handleDeletedNode, - drupalCreateNodeManifest, getExtendedFileNodeData, } from "./utils" const imageCdnDocs = `https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme` @@ -193,12 +192,7 @@ exports.sourceNodes = async ( pluginOptions.imageCDN = true } - const { - createNode, - setPluginStatus, - touchNode, - unstable_createNodeManifest, - } = actions + const { createNode, setPluginStatus, touchNode } = actions // Update the concurrency limit from the plugin options requestQueue.concurrency = concurrentAPIRequests @@ -746,12 +740,6 @@ ${JSON.stringify(webhookBody, null, 4)}` reporter ) - drupalCreateNodeManifest({ - attributes: datum?.attributes, - gatsbyNode: node, - unstable_createNodeManifest, - }) - nodes.set(node.id, node) }) } diff --git a/packages/gatsby-source-drupal/src/utils.ts b/packages/gatsby-source-drupal/src/utils.ts index 0084d47f11b0f..df05a01166130 100644 --- a/packages/gatsby-source-drupal/src/utils.ts +++ b/packages/gatsby-source-drupal/src/utils.ts @@ -9,9 +9,6 @@ import { import { getOptions } from "./plugin-options" -import { getGatsbyVersion } from "gatsby-core-utils" -import { lt, prerelease } from "semver" - function makeBackRefsKey(id) { return `backrefs-${id}` } @@ -314,7 +311,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)} ` ) - const { createNode, unstable_createNodeManifest } = actions + const { createNode } = actions const newNode = await nodeFromData( nodeToUpdate, @@ -325,12 +322,6 @@ ${JSON.stringify(nodeToUpdate, null, 4)} reporter ) - drupalCreateNodeManifest({ - attributes: nodeToUpdate.attributes, - gatsbyNode: newNode, - unstable_createNodeManifest, - }) - const nodesToUpdate = [newNode] const oldNodeReferencedNodes = await cache.get(makeRefNodesKey(newNode.id)) @@ -420,60 +411,6 @@ ${JSON.stringify(nodeToUpdate, null, 4)} } } -const GATSBY_VERSION_MANIFEST_V2 = `4.3.0` -const gatsbyVersion = - (typeof getGatsbyVersion === `function` && getGatsbyVersion()) || `0.0.0` -const gatsbyVersionIsPrerelease = prerelease(gatsbyVersion) -const shouldUpgradeGatsbyVersion = - lt(gatsbyVersion, GATSBY_VERSION_MANIFEST_V2) && !gatsbyVersionIsPrerelease - -let warnOnceForNoSupport = false -let warnOnceToUpgradeGatsby = false - -/** - * This fn creates node manifests which are used for Gatsby Cloud Previews via the Content Sync API/feature. - * Content Sync routes a user from Drupal to a page created from the entry data they're interested in previewing. - */ -export function drupalCreateNodeManifest({ - attributes, - gatsbyNode, - unstable_createNodeManifest, -}) { - const isPreview = - (process.env.NODE_ENV === `development` && - process.env.ENABLE_GATSBY_REFRESH_ENDPOINT) || - process.env.GATSBY_IS_PREVIEW === `true` - - const updatedAt = attributes?.revision_timestamp - const id = attributes?.drupal_internal__nid - const langcode = attributes?.langcode - - const supportsContentSync = typeof unstable_createNodeManifest === `function` - const shouldCreateNodeManifest = - id && updatedAt && supportsContentSync && isPreview - - if (shouldCreateNodeManifest) { - if (shouldUpgradeGatsbyVersion && !warnOnceToUpgradeGatsby) { - console.warn( - `Your site is doing more work than it needs to for Preview, upgrade to Gatsby ^${GATSBY_VERSION_MANIFEST_V2} for better performance` - ) - warnOnceToUpgradeGatsby = true - } - const manifestId = `${id}-${updatedAt}-${langcode}` - - unstable_createNodeManifest({ - manifestId, - node: gatsbyNode, - updatedAtUTC: updatedAt, - }) - } else if (!supportsContentSync && !warnOnceForNoSupport) { - warnOnceForNoSupport = true - console.warn( - `Drupal: Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` - ) - } -} - /** * This FN returns a Map with additional file node information that Drupal doesn't return on actual file nodes (namely the width/height of images) */ diff --git a/packages/gatsby-source-wordpress/src/steps/preview/index.ts b/packages/gatsby-source-wordpress/src/steps/preview/index.ts index c80910b64bb91..42ecdfeaa9a65 100644 --- a/packages/gatsby-source-wordpress/src/steps/preview/index.ts +++ b/packages/gatsby-source-wordpress/src/steps/preview/index.ts @@ -8,13 +8,11 @@ import PQueue from "p-queue" import { dump } from "dumper.js" import { actions as gatsbyActions } from "gatsby/dist/redux/actions/public" -import { remoteSchemaSupportsFieldNameOnTypeName } from "~/steps/ingest-remote-schema/introspect-remote-schema" import { paginatedWpNodeFetch } from "~/steps/source-nodes/fetch-nodes/fetch-nodes-paginated" import fetchGraphql from "~/utils/fetch-graphql" import { getStore } from "~/store" -import { fetchAndCreateSingleNode } from "~/steps/source-nodes/update-nodes/wp-actions/update" import { formatLogMessage } from "~/utils/format-log-message" import { touchValidNodes } from "../source-nodes/update-nodes/fetch-node-updates" @@ -279,26 +277,6 @@ export const sourcePreview = async ({ modified: previewData.modified, sendPreviewStatus, }) - - const { node } = await fetchAndCreateSingleNode({ - actionType: `PREVIEW`, - ...previewData, - previewParentId: previewData.parentDatabaseId, - isPreview: true, - }) - - if ( - previewData?.manifestIds?.length && - `unstable_createNodeManifest` in actions && - node - ) { - previewData.manifestIds.forEach(manifestId => { - actions.unstable_createNodeManifest({ - manifestId, - node, - }) - }) - } } /** @@ -358,12 +336,6 @@ export const sourcePreviews = async (helpers: GatsbyHelpers): Promise => { dump(webhookBody) } - const wpGatsbyPreviewNodeManifestsAreSupported = - await remoteSchemaSupportsFieldNameOnTypeName({ - typeName: `GatsbyPreviewData`, - fieldName: `manifestIds`, - }) - const previewActions = await paginatedWpNodeFetch({ contentTypePlural: `actionMonitorActions`, nodeTypeName: `ActionMonitor`, @@ -398,7 +370,6 @@ export const sourcePreviews = async (helpers: GatsbyHelpers): Promise => { remoteUrl singleName userDatabaseId - ${wpGatsbyPreviewNodeManifestsAreSupported ? `manifestIds` : ``} } } pageInfo { diff --git a/packages/gatsby/index.d.ts b/packages/gatsby/index.d.ts index 6b7af9f0a86b0..60c31230d8b5d 100644 --- a/packages/gatsby/index.d.ts +++ b/packages/gatsby/index.d.ts @@ -1380,17 +1380,6 @@ export interface Actions { plugin?: ActionPlugin ): void - /** @see https://www.gatsbyjs.com/docs/reference/config-files/actions/#unstable_createNodeManifest */ - unstable_createNodeManifest( - this: void, - args: { - manifestId: string - node: Node - updatedAtUTC?: string | number - }, - plugin?: ActionPlugin - ): void - /** @see https://www.gatsbyjs.com/docs/actions/#setRequestHeaders */ setRequestHeaders( this: void, diff --git a/packages/gatsby/src/query/query-watcher.ts b/packages/gatsby/src/query/query-watcher.ts index df3a69b1dbc34..80bb4321f1690 100644 --- a/packages/gatsby/src/query/query-watcher.ts +++ b/packages/gatsby/src/query/query-watcher.ts @@ -21,7 +21,6 @@ import { IGatsbyStaticQueryComponents } from "../redux/types" import queryCompiler from "./query-compiler" import report from "gatsby-cli/lib/reporter" import { getGatsbyDependents } from "../utils/gatsby-dependents" -import { processNodeManifests } from "../utils/node-manifest" const debug = require(`debug`)(`gatsby:query-watcher`) @@ -289,13 +288,6 @@ export const updateStateAndRunQueries = async ( `) } - - if (process.env.NODE_ENV === `development`) { - /** - * only process node manifests here in develop. we want this to run every time queries are updated. for gatsby build we process node manifests in src/utils/page-data.ts after all queries are run and pages are created. If we process node manifests in this location for gatsby build we wont have all the information needed to create the manifests. If we don't process manifests in this location during gatsby develop manifests will only be written once and never again when more manifests are created. - */ - await processNodeManifests() - } } export const extractQueries = ({ diff --git a/packages/gatsby/src/redux/actions/internal.ts b/packages/gatsby/src/redux/actions/internal.ts index d8ee3b3b2451c..db0ca46851db3 100644 --- a/packages/gatsby/src/redux/actions/internal.ts +++ b/packages/gatsby/src/redux/actions/internal.ts @@ -25,7 +25,6 @@ import { IQueryClearDirtyQueriesListToEmitViaWebsocket, ICreateJobV2FromInternalAction, ICreatePageDependencyActionPayloadType, - IDeleteNodeManifests, IClearGatsbyImageSourceUrlAction, } from "../types" @@ -366,12 +365,6 @@ export const setFunctions = ( } } -export const deleteNodeManifests = (): IDeleteNodeManifests => { - return { - type: `DELETE_NODE_MANIFESTS`, - } -} - export const createJobV2FromInternalJob = (internalJob: InternalJob): ICreateJobV2FromInternalAction => (dispatch, getState): Promise> => { diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index da30fddc21b56..16aefcfd32dce 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -163,7 +163,7 @@ const reservedFields = [ * @param {string} page.path Any valid URL. Must start with a forward slash. Unicode characters should be passed directly and not encoded (eg. `á` not `%C3%A1`). * @param {string} page.matchPath Path that Reach Router uses to match the page on the client side. * Also see docs on [matchPath](/docs/gatsby-internals-terminology/#matchpath) - * @param {string} page.ownerNodeId The id of the node that owns this page. This is used for routing users to previews via the unstable_createNodeManifest public action. Since multiple nodes can be queried on a single page, this allows the user to tell us which node is the main node for the page. Note that the ownerNodeId must be for a node which is queried on this page via a GraphQL query. + * @param {string} page.ownerNodeId The id of the node that owns this page. * @param {string} page.component The absolute path to the component for this page * @param {Object} page.context Context data for this page. Passed as props * to the component `this.props.pageContext` as well as to the graphql query @@ -1415,38 +1415,6 @@ actions.createServerVisitedPage = (chunkName: string) => { } } -/** - * Creates an individual node manifest. - * This is used to tie the unique revision state within a data source at the current point in time to a page generated from the provided node when it's node manifest is processed. - * - * @param {Object} manifest Manifest data - * @param {string} manifest.manifestId An id which ties the unique revision state of this manifest to the unique revision state of a data source. - * @param {Object} manifest.node The Gatsby node to tie the manifestId to. See the "createNode" action for more information about the node object details. - * @param {string} manifest.updatedAtUTC (optional) The time in which the node was last updated. If this parameter is not included, a manifest is created for every node that gets called. By default, node manifests are created for content updated in the last 30 days. To change this, set a `NODE_MANIFEST_MAX_DAYS_OLD` environment variable. - * @example - * unstable_createNodeManifest({ - * manifestId: `post-id-1--updated-53154315`, - * updatedAtUTC: `2021-07-08T21:52:28.791+01:00`, - * node: { - * id: `post-id-1` - * }, - * }) - */ -actions.unstable_createNodeManifest = ( - { manifestId, node, updatedAtUTC }, - plugin: Plugin -) => { - return { - type: `CREATE_NODE_MANIFEST`, - payload: { - manifestId, - node, - pluginName: plugin.name, - updatedAtUTC, - }, - } -} - /** * Marks a source plugin as "stateful" which disables automatically deleting untouched nodes. Stateful source plugins manage deleting their own nodes without stale node checks in Gatsby. * Enabling this is a major performance improvement for source plugins that manage their own node deletion. It also lowers the total memory required by a source plugin. diff --git a/packages/gatsby/src/redux/reducers/index.ts b/packages/gatsby/src/redux/reducers/index.ts index e2b394a5de986..ced276b4dac8d 100644 --- a/packages/gatsby/src/redux/reducers/index.ts +++ b/packages/gatsby/src/redux/reducers/index.ts @@ -30,7 +30,6 @@ import { visitedPagesReducer } from "./visited-page" import { htmlReducer } from "./html" import { functionsReducer } from "./functions" import { telemetryReducer } from "./telemetry" -import { nodeManifestReducer } from "./node-manifest" import { reducer as pageTreeReducer } from "gatsby-cli/lib/reporter/redux/reducers/page-tree" import { setRequestHeadersReducer } from "./set-request-headers" import { statefulSourcePluginsReducer } from "./stateful-source-plugins" @@ -75,7 +74,6 @@ export { queriesReducer as queries, htmlReducer as html, functionsReducer as functions, - nodeManifestReducer as nodeManifests, pageTreeReducer as pageTree, setRequestHeadersReducer as requestHeaders, statefulSourcePluginsReducer as statefulSourcePlugins, diff --git a/packages/gatsby/src/redux/reducers/node-manifest.ts b/packages/gatsby/src/redux/reducers/node-manifest.ts deleted file mode 100644 index cabbb312ca296..0000000000000 --- a/packages/gatsby/src/redux/reducers/node-manifest.ts +++ /dev/null @@ -1,80 +0,0 @@ -import reporter from "gatsby-cli/lib/reporter" - -import { - IGatsbyState, - ICreateNodeManifest, - IDeleteNodeManifests, -} from "../types" - -const ONE_DAY = 1000 * 60 * 60 * 24 // ms * sec * min * hr. -const DEFAULT_MAX_DAYS_OLD = 30 - -export const nodeManifestReducer = ( - state: IGatsbyState["nodeManifests"] = [], - action: ICreateNodeManifest | IDeleteNodeManifests -): IGatsbyState["nodeManifests"] => { - switch (action.type) { - case `CREATE_NODE_MANIFEST`: { - const { manifestId, pluginName, node, updatedAtUTC } = action.payload - - const maxDaysOld = - Number(process.env.NODE_MANIFEST_MAX_DAYS_OLD) || DEFAULT_MAX_DAYS_OLD - - if (updatedAtUTC) { - const nodeLastUpdatedAtUTC: number = new Date(updatedAtUTC).getTime() - if ( - (nodeLastUpdatedAtUTC as any) instanceof Date && - !isNaN(nodeLastUpdatedAtUTC) - ) { - reporter.warn( - `Plugin ${pluginName} called unstable_createNodeManifest with an updatedAtUTC that isn't a proper value to instantiate a Date.` - ) - - return state - } - - const shouldCreateNodeManifest = - Date.now() - nodeLastUpdatedAtUTC <= maxDaysOld * ONE_DAY - - if (!shouldCreateNodeManifest) { - return state - } - } - - if (typeof manifestId !== `string`) { - reporter.warn( - `Plugin ${pluginName} called unstable_createNodeManifest with a manifestId that isn't a string.` - ) - - return state - } - - if (!node?.id) { - reporter.warn( - `Plugin ${pluginName} called unstable_createNodeManifest but didn't provide a node.` - ) - - return state - } - - state.push({ - manifestId, - pluginName, - node: { - id: node.id, - }, - }) - - return state - } - - case `DELETE_NODE_MANIFESTS`: { - state = [] - - return state - } - - default: - return state - } -} diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts index 44e8abaafcfa2..81082c97b8174 100644 --- a/packages/gatsby/src/redux/types.ts +++ b/packages/gatsby/src/redux/types.ts @@ -289,7 +289,6 @@ export interface IGatsbyState { > typesToPlugins: Map } - nodeManifests: Array requestHeaders: Map statefulSourcePlugins: Set telemetry: ITelemetry @@ -1155,28 +1154,6 @@ interface ISSRUsedUnsafeBuiltin { type: `SSR_USED_UNSAFE_BUILTIN` } -export interface ICreateNodeManifest { - type: `CREATE_NODE_MANIFEST` - payload: { - manifestId: string - node: IGatsbyNode - pluginName: string - updatedAtUTC?: string | number - } -} - -export interface IDeleteNodeManifests { - type: `DELETE_NODE_MANIFESTS` -} - -export interface INodeManifest { - manifestId: string - pluginName: string - node: { - id: string - } -} - export interface ISetDomainRequestHeaders { type: `SET_REQUEST_HEADERS` payload: { diff --git a/packages/gatsby/src/utils/__tests__/node-manifest.ts b/packages/gatsby/src/utils/__tests__/node-manifest.ts deleted file mode 100644 index fed71187c620e..0000000000000 --- a/packages/gatsby/src/utils/__tests__/node-manifest.ts +++ /dev/null @@ -1,606 +0,0 @@ -import { foundPageByToLogIds } from "./../node-manifest" -import path from "path" -import fs from "fs-extra" -import reporter from "gatsby-cli/lib/reporter" -import { store } from "../../redux" -import { actions } from "../../redux/actions" -import { getNode } from "../../datastore" - -import { INodeManifest } from "./../../redux/types" - -import { - warnAboutNodeManifestMappingProblems, - processNodeManifests, -} from "../node-manifest" - -function itWindows(name: string, fn: () => void): void { - return process.platform === `win32` ? it(name, fn) : xit(name, fn) -} - -jest.mock(`fs-extra`, () => { - return { - ensureDir: jest.fn(), - writeJSON: jest.fn((manifestFilePath, finalManifest) => { - if (process.env.DEBUG) { - console.log({ manifestFilePath, finalManifest }) - } - return new Promise(resolve => - resolve({ - manifestFilePath, - finalManifest, - }) - ) - }), - } -}) - -jest.mock(`gatsby-cli/lib/reporter`, () => { - return { - error: jest.fn(input => { - if (process.env.DEBUG) { - console.error(JSON.stringify(input, null, 2)) - } - return input - }), - warn: jest.fn(message => { - if (process.env.DEBUG) { - console.warn(message) - } - return message - }), - info: jest.fn(message => { - if (process.env.DEBUG) { - console.info(message) - } - return message - }), - } -}) - -jest.mock(`../../datastore`, () => { - return { - getNode: jest.fn(), - } -}) - -const storeState = { - nodeManifests: [], - nodes: new Map(), - pages: new Map(), - program: { - directory: process.cwd(), - }, - queries: { byNode: new Map() }, -} - -jest.mock(`../../redux`, () => { - return { - emitter: { - on: jest.fn(), - }, - store: { - getState: jest.fn(), - dispatch: jest.fn(), - }, - } -}) - -function mockGetNodes(nodeStore: Map): void { - getNode.mockImplementation(id => nodeStore.get(id)) -} - -beforeEach(() => { - jest.clearAllMocks() - process.env.gatsby_log_level = null -}) - -describe(`processNodeManifests() warnings`, () => { - const useContextIdInsteadOfOwnerNodeId = { - pages: new Map([ - [ - `/test`, - { - path: `/test`, - context: { id: `1` }, - }, - ], - [ - `/test-2`, - { - path: `/test-2`, - context: { id: `2` }, - }, - ], - ]), - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - ], - queries: { - byNode: new Map([ - [`1`, new Set([`/test`, `/test-2`])], - [`2`, new Set([`/test`, `/test-2`])], - ]), - }, - } - - const multipleMappingProblems = { - pages: new Map([ - [ - `/test`, - { - path: `/test`, - context: { id: `1` }, - }, - ], - ]), - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - { - pluginName: `test`, - node: { id: `2` }, - manifestId: `2`, - }, - ], - } - - it(`logs warning code for not finding page`, async () => { - mockGetNodes(new Map([[`1`, { id: `1` }]])) - - store.getState.mockImplementation(() => { - return { - ...storeState, - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - ], - } - }) - - await processNodeManifests() - - expect(reporter.error).not.toBeCalled() - expect(reporter.info).toBeCalledWith( - expect.stringContaining( - `unstable_createNodeManifest produced warnings [${foundPageByToLogIds.none}]` - ) - ) - }) - - it(`logs warning code for finding page with context.id instead of ownerNodeId`, async () => { - mockGetNodes( - new Map([ - [`1`, { id: `1` }], - [`2`, { id: `2` }], - ]) - ) - store.getState.mockImplementation(() => { - return { - ...storeState, - ...useContextIdInsteadOfOwnerNodeId, - } - }) - - // first as develop - process.env.NODE_ENV = `development` - - await processNodeManifests() - - expect(reporter.error).not.toBeCalled() - expect(reporter.info).toBeCalledWith( - expect.stringContaining( - `unstable_createNodeManifest produced warnings [${ - foundPageByToLogIds[`context.id`] - }]` - ) - ) - }) - - it(`logs out list of warning codes for multiple mapping errors`, async () => { - mockGetNodes( - new Map([ - [`1`, { id: `1` }], - [`2`, { id: `2` }], - ]) - ) - store.getState.mockImplementation(() => { - return { - ...storeState, - ...multipleMappingProblems, - } - }) - - // first as develop - process.env.NODE_ENV = `development` - - await processNodeManifests() - - expect(reporter.error).not.toBeCalled() - expect(reporter.info).toBeCalledWith( - expect.stringContaining(foundPageByToLogIds.none) - ) - expect(reporter.info).toBeCalledWith( - expect.stringContaining(foundPageByToLogIds[`context.id`]) - ) - }) - - describe(`Verbose mode`, () => { - beforeEach(() => { - process.env.gatsby_log_level = `verbose` - }) - - it(`warns about no page found for manifest node id`, async () => { - mockGetNodes(new Map([[`1`, { id: `1` }]])) - - store.getState.mockImplementation(() => { - return { - ...storeState, - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - ], - } - }) - - await processNodeManifests() - - expect(reporter.error).toBeCalled() - expect(reporter.error).toBeCalledWith( - expect.objectContaining({ id: foundPageByToLogIds.none }) - ) - }) - - it(`warns about using context.id to map from node->page instead of ownerNodeId`, async () => { - mockGetNodes( - new Map([ - [`1`, { id: `1` }], - [`2`, { id: `2` }], - ]) - ) - store.getState.mockImplementation(() => { - return { - ...storeState, - ...useContextIdInsteadOfOwnerNodeId, - } - }) - - // first as develop - process.env.NODE_ENV = `development` - - await processNodeManifests() - - expect(reporter.error).toBeCalled() - expect(reporter.error).toBeCalledWith( - expect.objectContaining({ - id: foundPageByToLogIds[`context.id`], - }) - ) - - // then as build - process.env.NODE_ENV = `production` - - await processNodeManifests() - - expect(reporter.error).toBeCalled() - expect(reporter.error).toBeCalledWith( - expect.objectContaining({ - id: foundPageByToLogIds[`context.id`], - }) - ) - process.env.NODE_ENV = `test` - }) - - it(`warns about using node->query tracking to map from node->page instead of using ownerNodeId`, async () => { - mockGetNodes(new Map([[`1`, { id: `1` }]])) - store.getState.mockImplementation(() => { - return { - ...storeState, - pages: new Map([ - [ - `/test`, - { - path: `/test`, - context: {}, - }, - ], - ]), - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - ], - queries: { - byNode: new Map([[`1`, new Set([`/test`])]]), - }, - } - }) - - await processNodeManifests() - - expect(reporter.error).toBeCalled() - expect(reporter.error).toBeCalledWith( - expect.objectContaining({ - id: foundPageByToLogIds.queryTracking, - }) - ) - }) - - it(`doesn't warn when using the filesystem route api to map nodes->pages`, () => { - const { logId } = warnAboutNodeManifestMappingProblems({ - inputManifest: { - pluginName: `test`, - node: { id: `1` }, - manifestId: `1`, - }, - pagePath: `/test`, - foundPageBy: `filesystem-route-api`, - verbose: process.env.gatsby_log_level === `verbose`, - }) - - expect(reporter.error).not.toBeCalled() - expect(logId).toEqual(foundPageByToLogIds[`filesystem-route-api`]) - }) - }) -}) - -describe(`processNodeManifests`, () => { - it(`Doesn't do anything when there are no pending manifests`, async () => { - store.getState.mockImplementation(() => storeState) - - await processNodeManifests() - - expect(fs.writeJSON).not.toBeCalled() - expect(reporter.info).not.toBeCalled() - expect(reporter.warn).not.toBeCalled() - expect(reporter.error).not.toBeCalled() - expect(store.dispatch).not.toBeCalled() - }) - - const testProcessNodeManifestsWithUpdatedAt = async (): Promise => { - const today = new Date() - const twentyNineDaysAgoString = new Date( - new Date().setDate(today.getDate() - 29) - ).toISOString() - const thirtyOneDaysAgoString = new Date( - new Date().setDate(today.getDate() - 31) - ).toISOString() - - const nodes = [ - { id: `1`, usePageContextId: true, updatedAt: today.toISOString() }, - { id: `2`, useOwnerNodeId: true, updatedAt: twentyNineDaysAgoString }, - { id: `3`, useQueryTracking: true, updatedAt: thirtyOneDaysAgoString }, - ] - const { store } = jest.requireActual(`../../redux`) - - const createPayload = ( - id, - updatedAtUTC - ): { - manifestId: string - node: { id: string } - updatedAtUTC: string - } => { - return { - manifestId: `${id}-${updatedAtUTC}`, - node: { - id, - }, - updatedAtUTC, - } - } - - // Doesn't process manifest that is 31 days old - nodes.forEach(node => { - const payload = createPayload(node.id, node.updatedAt) - store.dispatch( - actions.unstable_createNodeManifest(payload, { - name: `gatsby-source-test`, - }) - ) - }) - - const { nodeManifests } = store.getState() - - process.env.NODE_MANIFEST_MAX_DAYS_OLD = `32` - - // Processes all three manifests - nodes.forEach(node => { - const payload = createPayload(node.id, node.updatedAt) - store.dispatch( - actions.unstable_createNodeManifest(payload, { - name: `gatsby-source-test`, - }) - ) - }) - - process.env.NODE_MANIFEST_MAX_DAYS_OLD = null - - expect(nodeManifests.length).toBe(5) - } - - const testProcessNodeManifests = async (): Promise => { - const nodes = [ - { id: `1`, usePageContextId: true }, - { id: `2`, useOwnerNodeId: true }, - { id: `3`, useQueryTracking: true }, - ] - mockGetNodes( - new Map(nodes.map(node => [`${node.id}`, { id: `${node.id}` }])) - ) - - const pendingManifests: Array = [ - ...nodes, - { - // this node doesn't exist - id: `4`, - }, - ].map(node => { - return { - pluginName: `test`, - manifestId: `${node.id}`, - node, - } - }) - - store.getState.mockImplementation(() => { - return { - ...storeState, - pages: new Map( - nodes.map(node => [ - `/${node.id}`, - { - path: `/${node.id}`, - ownerNodeId: node.useOwnerNodeId ? node.id : null, - context: { - id: node.usePageContextId ? node.id : null, - }, - }, - ]) - ), - nodeManifests: pendingManifests, - queries: { - byNode: new Map( - nodes.map(node => [`${node.id}`, new Set([`/${node.id}`])]) - ), - }, - } - }) - - await processNodeManifests() - - if (process.env.gatsby_log_level === `verbose`) { - expect(reporter.error).toBeCalled() - expect(reporter.error).toBeCalledWith({ - context: { nodeId: `4`, pluginName: `test` }, - id: `11804`, - }) - } else { - expect(reporter.error).not.toBeCalled() - expect(reporter.info).toBeCalledWith(expect.stringContaining(`11804`)) - } - - expect(reporter.info).toBeCalled() - expect(reporter.info).toBeCalledWith( - expect.stringContaining( - `Wrote out ${nodes.length} node page manifest files` - ) - ) - expect(reporter.info).toBeCalledWith( - expect.stringContaining(`1 manifest couldn't be processed`) - ) - expect(store.dispatch).toBeCalled() - - expect(fs.ensureDir).toBeCalledTimes(nodes.length) - expect(fs.writeJSON).toBeCalledTimes(nodes.length) - - pendingManifests.forEach((manifest, index) => { - // if a node doesn't exist for this manifest we don't want to assert that - // a manifest was written - if (!nodes.find(node => node.id === manifest.node.id)) { - return - } - - expect(fs.writeJSON).toHaveBeenNthCalledWith( - index + 1, - `${path.join( - process.cwd(), - `public`, - `__node-manifests`, - `test`, - manifest.manifestId - )}.json`, - expect.objectContaining({ - page: { - path: `/${manifest.node.id}`, - }, - }) - ) - }) - } - - it(`processes node manifests gatsby develop`, async () => { - process.env.NODE_ENV = `development` - await testProcessNodeManifests() - process.env.NODE_ENV = `test` - }) - - it(`processes node manifests gatsby build`, async () => { - process.env.NODE_ENV = `production` - process.env.gatsby_log_level = `verbose` - await testProcessNodeManifests() - process.env.NODE_ENV = `test` - }) - - it(`creates manifests only for recently updated manifests`, async () => { - process.env.NODE_ENV = `production` - await testProcessNodeManifestsWithUpdatedAt() - process.env.NODE_ENV = `test` - }) - itWindows(`replaces reserved Windows characters with a dash`, async () => { - const nodes = [ - { id: `1`, usePageContextId: true }, - { id: `2`, useOwnerNodeId: true }, - { id: `3`, useQueryTracking: true }, - ] - mockGetNodes( - new Map(nodes.map(node => [`${node.id}`, { id: `${node.id}` }])) - ) - - store.getState.mockImplementation(() => { - return { - ...storeState, - pages: new Map([ - [ - `/test`, - { - path: `/test`, - context: { id: `1` }, - }, - ], - [ - `/test-2`, - { - path: `/test-2`, - context: { id: `2` }, - }, - ], - ]), - nodeManifests: [ - { - pluginName: `test`, - node: { id: `1` }, - // A manifest id containing all of the reserved windows characters that we check - // for and replace - manifestId: `\\*"<>/:?|`, - }, - ], - queries: { - byNode: new Map([ - [`1`, new Set([`/test`, `/test-2`])], - [`2`, new Set([`/test`, `/test-2`])], - ]), - }, - } - }) - - await processNodeManifests() - const nodeManifestFileName = path.basename(fs.writeJSON.mock.calls[0][0]) - - expect(nodeManifestFileName).toEqual(`---------.json`) - }) -}) diff --git a/packages/gatsby/src/utils/node-manifest.ts b/packages/gatsby/src/utils/node-manifest.ts deleted file mode 100644 index 55ac85d893a7a..0000000000000 --- a/packages/gatsby/src/utils/node-manifest.ts +++ /dev/null @@ -1,513 +0,0 @@ -import type { ErrorId } from "gatsby-cli/lib/structured-errors/error-map" -import { getNode } from "./../datastore" -import { IGatsbyNode, IGatsbyPage, INodeManifest } from "./../redux/types" -import reporter from "gatsby-cli/lib/reporter" -import { store } from "../redux/" -import { internalActions } from "../redux/actions" -import path from "path" -import fs from "fs-extra" -import fastq from "fastq" - -interface INodeManifestPage { - path?: string -} - -/** - * This it the output after processing calls to the public unstable_createNodeManifest action - */ -interface INodeManifestOut { - page: INodeManifestPage - node: { - id: string - } - foundPageBy: FoundPageBy -} - -type FoundPageBy = - | `ownerNodeId` - | `filesystem-route-api` - // for these three we warn to use ownerNodeId instead - | `context.id` - | `context.slug` - | `queryTracking` - | `none` - -type PreviouslyWrittenNodeManifests = Map> - -function getNodeManifestFileLimit(): number { - const defaultLimit = 10000 - - const overrideLimit = - process.env.NODE_MANIFEST_FILE_LIMIT && - Number(process.env.NODE_MANIFEST_FILE_LIMIT) - - return overrideLimit || defaultLimit -} -/** - * This defines a limit to the number number of node manifest files that will be written to disk - */ -const NODE_MANIFEST_FILE_LIMIT = getNodeManifestFileLimit() - -/** - * Finds a final built page by nodeId or by node.slug as a fallback. - * - * Note that this function wont work properly in `gatsby develop` - * since develop no longer runs all page queries when creating pages. - * We use the node id to query mapping to find the right page but - * this mapping only exists once you've visited a page in your browser. - * When this fn is being used for routing to previews the user wont necessarily have - * visited the page in the browser yet. - */ -async function findPageOwnedByNode({ - nodeId, - fullNode, - slug, -}: { - nodeId: string - fullNode: IGatsbyNode - slug?: string -}): Promise<{ - page: INodeManifestPage - foundPageBy: FoundPageBy -}> { - const state = store.getState() - const { pages, staticQueryComponents } = state - const { byNode, byConnection, trackedComponents } = state.queries - - const nodeType = fullNode?.internal?.type - - const firstPagePathWithNodeAsDataDependency = - // the first page found in node id to page query path tracking - byNode?.get(nodeId)?.values()?.next()?.value - - const firstPagePathWithNodeInGraphQLListField = - // the first page that queries for a list of this node type. - // we don't currently store a list of node ids for connection fields to queries - // we just store the query id or page path mapped to the connected GraphQL typename. - byConnection?.get(nodeType)?.values()?.next()?.value - - let pagePath = - firstPagePathWithNodeAsDataDependency || - firstPagePathWithNodeInGraphQLListField - - // for static queries, we can only find the first page using that static query - // the reason we would find `sq--` here is because byConnection (above) can return a page path or a static query ID (which starts with `sq--`) - if (pagePath?.startsWith(`sq--`)) { - const staticQueryComponentPath = - staticQueryComponents?.get(pagePath)?.componentPath - - const firstPagePathUsingStaticQueryComponent: string | null = - staticQueryComponentPath - ? trackedComponents - ?.get(staticQueryComponentPath) - ?.pages?.values() - ?.next()?.value - : null - - pagePath = firstPagePathUsingStaticQueryComponent - } - - let foundPageBy: FoundPageBy = pagePath ? `queryTracking` : `none` - - if (pages) { - let ownerPagePath: string | undefined - let foundOwnerNodeId = false - - // for each page this nodeId is queried in - for (const pageObject of pages.values()) { - // if we haven't found a page with this nodeId - // set as page.ownerNodeId then run this logic. - // this condition is on foundOwnerNodeId instead of ownerPagePath - // in case we find a page with the nodeId in page.context.id/context.slug - // and then later in the loop there's a page with this nodeId - // set on page.ownerNodeId. - // We always want to prefer ownerPagePath over context.id/context.slug - if (foundOwnerNodeId) { - break - } - - const path = pageObject.path - - const fullPage: IGatsbyPage | undefined = pages.get(path) - - foundOwnerNodeId = fullPage?.ownerNodeId === nodeId - - const foundPageIdInContext = fullPage?.context?.id === nodeId - - // querying by node.slug in GraphQL queries is common enough that we can search for it as a fallback after ownerNodeId, filesystem routes, and context.id - const foundPageSlugInContext = slug && fullPage?.context?.slug === slug - - if (foundOwnerNodeId) { - foundPageBy = `ownerNodeId` - } else if (foundPageIdInContext && fullPage) { - const pageCreatedByPluginName = getNode(fullPage.pluginCreatorId)?.name - - const pageCreatedByFilesystemPlugin = - pageCreatedByPluginName === `gatsby-plugin-page-creator` - - foundPageBy = pageCreatedByFilesystemPlugin - ? `filesystem-route-api` - : `context.id` - } else if (foundPageSlugInContext && fullPage) { - foundPageBy = `context.slug` - } - - if ( - fullPage && - // first check for the ownerNodeId on the page. this is - // the defacto owner. Can't get more specific than this - (foundOwnerNodeId || - // if there's no specified owner look to see if - // pageContext has an `id` variable which matches our - // nodeId. Using an "id" as a variable in queries is common - // and if we don't have an owner this is a better guess - // of an owner than grabbing the first page query we find - // that's mapped to this node id. - // this also makes this work with the filesystem Route API without - // changing that API. - foundPageIdInContext || - foundPageSlugInContext) - ) { - // save this path to use in our manifest! - ownerPagePath = fullPage.path - } - } - - if (ownerPagePath) { - pagePath = ownerPagePath - } - } - - return { - page: { - path: pagePath || null, - }, - foundPageBy, - } -} - -// these id's correspond to error id's in -// packages/gatsby-cli/src/structured-errors/error-map.ts -export const foundPageByToLogIds = { - none: `11801`, - [`context.id`]: `11802`, - [`context.slug`]: `11805`, - queryTracking: `11803`, - [`filesystem-route-api`]: `success`, - ownerNodeId: `success`, -} - -/** - * Takes in some info about a node manifest and the page we did or didn't find for it, then warns and returns the warning string - */ -export function warnAboutNodeManifestMappingProblems({ - inputManifest, - pagePath, - foundPageBy, - verbose, -}: { - inputManifest: INodeManifest - pagePath?: string - foundPageBy: FoundPageBy - verbose: boolean -}): { logId: string } { - let logId: ErrorId | `success` - - switch (foundPageBy) { - case `none`: - case `context.id`: - case `context.slug`: - case `queryTracking`: { - logId = foundPageByToLogIds[foundPageBy] - if (verbose) { - reporter.error({ - id: logId, - context: { - inputManifest, - pagePath, - }, - }) - } - break - } - - case `filesystem-route-api`: - case `ownerNodeId`: - logId = `success` - break - - default: { - throw Error(`Node Manifest mapping is in an impossible state`) - } - } - - return { - logId, - } -} - -/** - * Prepares and then writes out an individual node manifest file to be used for routing to previews. Manifest files are added via the public unstable_createNodeManifest action - */ -export async function processNodeManifest( - inputManifest: INodeManifest, - listOfUniqueErrorIds: Set, - nodeManifestPagePathMap: Map, - verboseLogs: boolean, - previouslyWrittenNodeManifests: PreviouslyWrittenNodeManifests -): Promise { - const nodeId = inputManifest.node.id - const fullNode = getNode(nodeId) - const noNodeWarningId = `11804` - - if (!fullNode) { - if (verboseLogs) { - reporter.error({ - id: noNodeWarningId, - context: { - pluginName: inputManifest.pluginName, - nodeId, - }, - }) - } else { - listOfUniqueErrorIds.add(noNodeWarningId) - } - - return null - } - - // map the node to a page that was created - const { page: nodeManifestPage, foundPageBy } = await findPageOwnedByNode({ - nodeId, - fullNode, - // querying by node.slug in GraphQL queries is common enough that we can search for it as a fallback after ownerNodeId, filesystem routes, and context.id - slug: fullNode?.slug as string, - }) - - const nodeManifestMappingProblemsContext = { - inputManifest, - pagePath: nodeManifestPage.path, - foundPageBy, - verbose: verboseLogs, - } - - if (verboseLogs) { - warnAboutNodeManifestMappingProblems(nodeManifestMappingProblemsContext) - } else { - const { logId } = warnAboutNodeManifestMappingProblems( - nodeManifestMappingProblemsContext - ) - - if (logId !== `success`) { - listOfUniqueErrorIds.add(logId) - } - } - - const finalManifest: INodeManifestOut = { - node: inputManifest.node, - page: nodeManifestPage, - foundPageBy, - } - - const gatsbySiteDirectory = store.getState().program.directory - - let fileNameBase = inputManifest.manifestId - - /** - * Windows has a handful of special/reserved characters that are not valid in a file path - * @reference https://superuser.com/questions/358855/what-characters-are-safe-in-cross-platform-file-names-for-linux-windows-and-os - * - * The two exceptions to the list linked above are - * - the colon that is part of the hard disk partition name at the beginning of a file path (i.e. C:) - * - backslashes. We don't want to replace backslashes because those are used to delineate what the actual file path is - * - * During local development, node manifests can be written to disk but are generally unused as they are only used - * for Content Sync which runs in Gatsby Cloud. Gatsby cloud is a Linux environment in which these special chars are valid in - * filepaths. To avoid errors on Windows, we replace all instances of the special chars in the filepath (with the exception of the - * hard disk partition name) with "-" to ensure that local Windows development setups do not break when attempting - * to write one of these manifests to disk. - */ - if (process.platform === `win32`) { - fileNameBase = fileNameBase.replace(/:|\/|\*|\?|"|<|>|\||\\/g, `-`) - } - - // write out the manifest file - const manifestFilePath = path.join( - gatsbySiteDirectory, - `public`, - `__node-manifests`, - inputManifest.pluginName, - `${fileNameBase}.json` - ) - - const manifestFileDir = path.dirname(manifestFilePath) - - await fs.ensureDir(manifestFileDir) - - const previouslyWrittenNodeManifest = - await previouslyWrittenNodeManifests.get(inputManifest.manifestId) - - // write a manifest if we don't currently have one written for this ID - // or if we can replace the written one with a manifest that has found a page - // NOTE: We still want to write out a manifest if foundPageBy is "none", this helps with error messaging - // But we prefer to write a manifest that has a foundPageBy that is NOT "none" - const shouldWriteManifest = - !previouslyWrittenNodeManifest || - (previouslyWrittenNodeManifest?.foundPageBy === `none` && - finalManifest.foundPageBy !== `none`) - - if (shouldWriteManifest) { - const writePromise = fs.writeJSON(manifestFilePath, finalManifest) - - // This prevents two manifests from writing to the same file at the same time - previouslyWrittenNodeManifests.set( - inputManifest.manifestId, - new Promise(resolve => { - writePromise.then(() => { - resolve(finalManifest) - }) - }) - ) - - await writePromise - } - - if (shouldWriteManifest && verboseLogs) { - reporter.info( - `Plugin ${inputManifest.pluginName} created a manifest with the id ${fileNameBase}` - ) - } else if (verboseLogs) { - reporter.info( - `Plugin ${inputManifest.pluginName} created a manifest with the id ${fileNameBase} but it was not written to disk because it was already written to disk previously.` - ) - } - - if (nodeManifestPage.path) { - nodeManifestPagePathMap.set(nodeManifestPage.path, fileNameBase) - } - - return finalManifest -} - -function nodeManifestSortComparerAscendingUpdatedAt(a, b): number { - /** - * Prioritize node manifests that have an updatedAtUTC so that manifests known to be - * newest are written to disk first. If neither have an updatedAtUTC, there isn't - * anything to sort - */ - if (!a.updatedAtUTC && !b.updatedAtUTC) { - return 0 - } - - if (!a.updatedAtUTC) { - return 1 - } - - if (!b.updatedAtUTC) { - return -1 - } - - return Date.parse(a.updatedAtUTC) - Date.parse(b.updatedAtUTC) -} - -/** - * Grabs all pending node manifests, processes them, writes them to disk, - * and then removes them from the store. - * Manifest files are added via the public unstable_createNodeManifest action in sourceNodes - */ -export async function processNodeManifests(): Promise | null> { - const verboseLogs = - process.env.gatsby_log_level === `verbose` || - process.env.VERBOSE_NODE_MANIFEST === `true` - - const startTime = Date.now() - let { nodeManifests } = store.getState() - - const totalManifests = nodeManifests.length - - if (totalManifests === 0) { - return null - } - - let totalProcessedManifests = 0 - let totalFailedManifests = 0 - const nodeManifestPagePathMap: Map = new Map() - const listOfUniqueErrorIds: Set = new Set() - const previouslyWrittenNodeManifests: PreviouslyWrittenNodeManifests = - new Map() - - async function processNodeManifestTask( - manifest: INodeManifest, - cb: fastq.done - ): Promise { - const processedManifest = await processNodeManifest( - manifest, - listOfUniqueErrorIds, - nodeManifestPagePathMap, - verboseLogs, - previouslyWrittenNodeManifests - ) - - if (processedManifest) { - totalProcessedManifests++ - } else { - totalFailedManifests++ - } - - // `setImmediate` below is a workaround against stack overflow - // occurring when there are many manifests - setImmediate(() => cb(null, true)) - return - } - - const processNodeManifestQueue = fastq(processNodeManifestTask, 25) - - if (totalManifests > NODE_MANIFEST_FILE_LIMIT) { - nodeManifests = [...nodeManifests] - nodeManifests.sort(nodeManifestSortComparerAscendingUpdatedAt) - nodeManifests = nodeManifests.slice(0, NODE_MANIFEST_FILE_LIMIT) - } - - for (const manifest of nodeManifests) { - processNodeManifestQueue.push(manifest, () => {}) - } - - if (!processNodeManifestQueue.idle()) { - await new Promise(resolve => { - processNodeManifestQueue.drain = resolve as () => unknown - }) - } - - const pluralize = (length: number): string => - length > 1 || length === 0 ? `s` : `` - - const endTime = Date.now() - - reporter.info( - `Wrote out ${totalProcessedManifests} node page manifest file${pluralize( - totalProcessedManifests - )} in ${endTime - startTime} ms. ${ - totalFailedManifests > 0 - ? `. ${totalFailedManifests} manifest${pluralize( - totalFailedManifests - )} couldn't be processed.` - : `` - }` - ) - - reporter.info( - (!verboseLogs && listOfUniqueErrorIds.size > 0 - ? `unstable_createNodeManifest produced warnings [${[ - ...listOfUniqueErrorIds, - ].join(`, `)}]. ` - : ``) + - `To see full warning messages set process.env.VERBOSE_NODE_MANIFEST to "true".\nVisit https://gatsby.dev/nodemanifest for more info on Node Manifests.` - ) - - // clean up all pending manifests from the store - store.dispatch(internalActions.deleteNodeManifests()) - return nodeManifestPagePathMap -} diff --git a/packages/gatsby/src/utils/page-data.ts b/packages/gatsby/src/utils/page-data.ts index c3b055d2a0e81..fca94a937da05 100644 --- a/packages/gatsby/src/utils/page-data.ts +++ b/packages/gatsby/src/utils/page-data.ts @@ -19,7 +19,6 @@ import { import { Span } from "opentracing" export { reverseFixedPagePath } -import { processNodeManifests } from "../utils/node-manifest" import { IExecutionResult } from "../query/types" import { getPageMode } from "./page-mode" import { ICollectedSlices } from "./babel/find-slices" @@ -182,9 +181,6 @@ export function isFlushEnqueued(): boolean { return isFlushPending } -let staleNodeManifests = false -const maxManifestIdsToLog = 50 - type IDataTask = | { type: "page" @@ -211,41 +207,12 @@ export async function flush(parentSpan?: Span): Promise { queries, slices, slicesByTemplate, - nodeManifests, } = store.getState() const isBuild = program?._?.[0] !== `develop` const { pagePaths, sliceNames } = pendingPageDataWrites let writePageDataActivity - let nodeManifestPagePathMap - - if (pagePaths.size > 0) { - // we process node manifests in this location because we need to add the manifestId to the page data. - // We use this manifestId to determine if the page data is up to date when routing. Here we create a map of "pagePath": "manifestId" while processing and writing node manifest files. - // We only do this when there are pending page-data writes because otherwise we could flush pending createNodeManifest calls before page-data.json files are written. Which means those page-data files wouldn't have the corresponding manifest id's written to them. - nodeManifestPagePathMap = await processNodeManifests() - } else if (nodeManifests.length > 0 && staleNodeManifests) { - staleNodeManifests = false - - reporter.warn( - `[gatsby] node manifests were created but no page-data.json files were written, so manifest ID's were not added to page-data.json files. This may be a bug or it may be due to a source plugin creating a node manifest for a node that did not change. Node manifest IDs: ${nodeManifests - .map(n => n.manifestId) - .slice(0, maxManifestIdsToLog) - .join(`,`)}${ - nodeManifests.length > maxManifestIdsToLog - ? ` There were ${ - nodeManifests.length - maxManifestIdsToLog - } additional ID's that were not logged due to output length.` - : `` - }` - ) - - nodeManifestPagePathMap = await processNodeManifests() - } else if (nodeManifests.length > 0) { - staleNodeManifests = true - } - if (pagePaths.size > 0 || sliceNames.size > 0) { writePageDataActivity = reporter.createProgress( `Writing page-data.json and slice-data.json files to public directory`, @@ -270,10 +237,6 @@ export async function flush(parentSpan?: Span): Promise { // them, a page might not exist anymore щ（ﾟДﾟщ） // This is why we need this check if (page) { - if (page.path && nodeManifestPagePathMap) { - page.manifestId = nodeManifestPagePathMap.get(page.path) - } - if (!isBuild && process.env.GATSBY_QUERY_ON_DEMAND) { // check if already did run query for this page // with query-on-demand we might have pending page-data write due to From 8fc361f175a0853629dde491ce143ae704303307 Mon Sep 17 00:00:00 2001 From: Mateusz Bocian Date: Tue, 25 Feb 2025 10:05:04 -0500 Subject: [PATCH 2/2] test: pin pnpm version to 9 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e06bf4b4466da..04d6244537f4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -386,6 +386,7 @@ jobs: image: "18.12.0" steps: - checkout + - run: npm i -g pnpm@latest-9 - run: ./scripts/assert-changed-files.sh "packages/*|.circleci/*" - <<: *attach_to_bootstrap - run: