Skip to content

Commit 6aa2dc6

Browse files
trekstyfle
trek
and
styfle
authored
Connecting Datadog / GH Actions / Turbo / Jest to detect flakey tests (#10334)
Co-authored-by: Trek Glowacki <[email protected]> Co-authored-by: Steven <[email protected]>
1 parent 9db3794 commit 6aa2dc6

File tree

26 files changed

+133
-42
lines changed

26 files changed

+133
-42
lines changed

‎.changeset/smart-walls-kneel.md

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
---

‎.github/workflows/test.yml

+8-2
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ jobs:
4747
token: ${{ secrets.GITHUB_TOKEN }}
4848
max_timeout: 360
4949
check_interval: 5
50-
5150
test:
5251
timeout-minutes: 120
5352
runs-on: ${{ matrix.runner }}
@@ -75,7 +74,6 @@ jobs:
7574
run: npm i -g [email protected]
7675

7776
- run: pnpm install
78-
7977
- name: fetch ssl certificate before build (linux, os x)
8078
if: matrix.runner != 'windows-latest'
8179
run: echo | openssl s_client -showcerts -servername 'api.vercel.com' -connect 76.76.21.21:443
@@ -88,6 +86,7 @@ jobs:
8886
run: node utils/gen.js && node_modules/.bin/turbo run test --cache-dir=".turbo" --log-order=stream --scope=${{matrix.packageName}} --no-deps -- ${{ join(matrix.testPaths, ' ') }}
8987
shell: bash
9088
env:
89+
JEST_JUNIT_OUTPUT_FILE: ${{github.workspace}}/.junit-reports/${{matrix.scriptName}}-${{matrix.packageName}}-${{matrix.chunkNumber}}-${{ matrix.runner }}.xml
9190
VERCEL_CLI_VERSION: ${{ needs.setup.outputs.dplUrl }}/tarballs/vercel.tgz
9291
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
9392
VERCEL_TEST_REGISTRATION_URL: ${{ secrets.VERCEL_TEST_REGISTRATION_URL }}
@@ -97,6 +96,13 @@ jobs:
9796
if: matrix.runner != 'windows-latest'
9897
run: echo | openssl s_client -showcerts -servername 'api.vercel.com' -connect 76.76.21.21:443
9998

99+
- name: 'Upload Test Report to Datadog'
100+
if: always()
101+
run: 'npx @datadog/[email protected] junit upload --service vercel-cli .junit-reports'
102+
env:
103+
DATADOG_API_KEY: ${{secrets.DATADOG_API_KEY_CLI}}
104+
DD_ENV: ci
105+
100106
summary:
101107
name: Summary
102108
runs-on: ubuntu-latest

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ __pycache__
3131
.turbo
3232
.eslintcache
3333
turbo-cache-key.json
34+
junit.xml

‎examples/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"scripts": {
66
"test-unit": "pnpm test __tests__/unit/",
77
"test-e2e": "pnpm test __tests__/integration/",
8-
"test": "jest --env node --verbose --runInBand --bail"
8+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail"
99
},
1010
"devDependencies": {
1111
"@types/jest": "27.4.1",

‎internals/get-package-json/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
],
1010
"scripts": {
1111
"build": "tsc",
12-
"test": "jest --env node --verbose --runInBand --bail",
12+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1313
"test-unit": "pnpm test tests/unit"
1414
},
1515
"devDependencies": {

‎packages/build-utils/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
},
1313
"scripts": {
1414
"build": "node build",
15-
"test": "jest --env node --verbose --runInBand --bail",
15+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1616
"test-unit": "pnpm test test/unit.*test.*",
1717
"test-e2e": "pnpm test test/integration.test.ts"
1818
},
@@ -44,6 +44,7 @@
4444
"glob": "8.0.3",
4545
"ignore": "4.0.6",
4646
"into-stream": "5.0.0",
47+
"jest-junit": "16.0.0",
4748
"js-yaml": "3.13.1",
4849
"minimatch": "3.1.2",
4950
"multistream": "2.1.1",

‎packages/cli/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"directory": "packages/cli"
1212
},
1313
"scripts": {
14-
"test": "jest --env node --verbose --bail",
14+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail",
1515
"test-unit": "pnpm test test/unit/",
1616
"test-e2e": "rimraf test/fixtures/integration && pnpm test test/integration-1.test.ts test/integration-2.test.ts test/integration-3.test.ts",
1717
"test-dev": "pnpm test test/dev/",
@@ -134,6 +134,7 @@
134134
"is-port-reachable": "3.1.0",
135135
"is-url": "1.2.2",
136136
"jaro-winkler": "0.2.8",
137+
"jest-junit": "16.0.0",
137138
"jest-matcher-utils": "29.3.1",
138139
"jsonlines": "0.1.1",
139140
"line-async-iterator": "3.0.0",

‎packages/client/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
"scripts": {
1717
"build": "tsc",
1818
"test-e2e": "pnpm test tests/create-deployment.test.ts tests/create-legacy-deployment.test.ts tests/paths.test.ts",
19-
"test": "jest --env node --verbose --runInBand --bail",
19+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
2020
"test-unit": "pnpm test tests/unit.*test.*"
2121
},
2222
"engines": {
@@ -32,6 +32,7 @@
3232
"@types/node-fetch": "2.5.4",
3333
"@types/recursive-readdir": "2.2.0",
3434
"@types/tar-fs": "1.16.1",
35+
"jest-junit": "16.0.0",
3536
"typescript": "4.9.5"
3637
},
3738
"dependencies": {

‎packages/edge/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@
1616
},
1717
"scripts": {
1818
"build": "tsup src/index.ts --dts --format esm,cjs",
19-
"test": "jest --env node --verbose --runInBand --bail",
19+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
2020
"test-unit": "pnpm test",
2121
"build:docs": "typedoc && node scripts/fix-links.js && prettier --write docs/**/*.md docs/*.md"
2222
},
2323
"devDependencies": {
2424
"@edge-runtime/jest-environment": "2.2.3",
2525
"@types/jest": "27.4.1",
26+
"jest-junit": "16.0.0",
2627
"ts-node": "8.9.1",
2728
"tsup": "6.1.2",
2829
"typedoc": "0.24.6",

‎packages/error-utils/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@
1111
},
1212
"scripts": {
1313
"build": "tsc",
14-
"test": "jest --coverage --env node --verbose",
14+
"test": "jest --reporters=default --reporters=jest-junit --coverage --env node --verbose",
1515
"test-unit": "pnpm test"
1616
},
1717
"license": "Apache-2.0",
1818
"devDependencies": {
1919
"@types/jest": "29.2.1",
2020
"@types/node": "14.18.33",
21+
"jest-junit": "16.0.0",
2122
"typescript": "4.9.5"
2223
}
2324
}

‎packages/frameworks/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"license": "Apache-2.0",
1515
"scripts": {
1616
"build": "tsc",
17-
"test": "jest --env node --verbose --runInBand --bail",
17+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1818
"test-unit": "pnpm test"
1919
},
2020
"dependencies": {
@@ -29,6 +29,7 @@
2929
"@vercel/error-utils": "1.0.10",
3030
"@vercel/routing-utils": "2.2.1",
3131
"ajv": "6.12.2",
32+
"jest-junit": "16.0.0",
3233
"typescript": "4.9.5"
3334
}
3435
}

‎packages/fs-detectors/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"license": "Apache-2.0",
1616
"scripts": {
1717
"build": "tsc",
18-
"test": "jest --env node --verbose --runInBand --bail",
18+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1919
"test-unit": "pnpm test test/unit.*test.*",
2020
"test-e2e": "pnpm test test/integration.test.ts"
2121
},
@@ -37,6 +37,7 @@
3737
"@types/node": "14.18.33",
3838
"@types/semver": "7.3.10",
3939
"@vercel/build-utils": "6.8.3",
40+
"jest-junit": "16.0.0",
4041
"typescript": "4.9.5"
4142
}
4243
}

‎packages/gatsby-plugin-vercel-analytics/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"license": "Apache-2.0",
3535
"devDependencies": {
3636
"@babel/cli": "7.20.7",
37-
"@babel/core": "7.5.0"
37+
"@babel/core": "7.5.0",
38+
"jest-junit": "16.0.0"
3839
}
3940
}

‎packages/gatsby-plugin-vercel-builder/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"@types/fs-extra": "11.0.1",
3333
"@types/node": "14.18.33",
3434
"@types/react": "18.0.26",
35+
"jest-junit": "16.0.0",
3536
"typescript": "4.9.5"
3637
}
3738
}

‎packages/go/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
"scripts": {
1313
"build": "node build",
14-
"test": "jest --env node --verbose --runInBand --bail",
14+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1515
"test-e2e": "pnpm test"
1616
},
1717
"files": [
@@ -32,6 +32,7 @@
3232
"async-retry": "1.3.1",
3333
"execa": "^1.0.0",
3434
"fs-extra": "^7.0.0",
35+
"jest-junit": "16.0.0",
3536
"node-fetch": "^2.2.1",
3637
"string-argv": "0.3.1",
3738
"tar": "4.4.6",

‎packages/hydrogen/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"scripts": {
1313
"build": "node build.js",
1414
"test-e2e": "pnpm test test/test.js",
15-
"test": "jest --env node --verbose --bail --runInBand"
15+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand"
1616
},
1717
"files": [
1818
"dist",
@@ -25,6 +25,7 @@
2525
"@vercel/static-config": "2.0.17",
2626
"execa": "3.2.0",
2727
"fs-extra": "11.1.0",
28+
"jest-junit": "16.0.0",
2829
"ts-morph": "12.0.0"
2930
}
3031
}

‎packages/next/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"scripts": {
88
"build": "node build.js",
99
"build-dev": "node build.js --dev",
10-
"test": "jest --env node --verbose --bail --runInBand --testTimeout=360000",
10+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand --testTimeout=360000",
1111
"test-unit": "pnpm test test/unit/",
1212
"test-next-local": "pnpm test test/integration/*.test.js test/integration/*.test.ts",
1313
"test-next-local:middleware": "pnpm test test/integration/middleware.test.ts",
@@ -49,6 +49,7 @@
4949
"find-up": "4.1.0",
5050
"fs-extra": "7.0.0",
5151
"get-port": "5.0.0",
52+
"jest-junit": "16.0.0",
5253
"nanoid": "3.3.4",
5354
"ndjson": "2.0.0",
5455
"pretty-bytes": "5.3.0",

‎packages/node/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"etag": "1.8.1",
5656
"execa": "3.2.0",
5757
"fs-extra": "11.1.0",
58+
"jest-junit": "16.0.0",
5859
"source-map-support": "0.5.12",
5960
"test-listen": "1.1.0"
6061
}

‎packages/python/package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
},
1616
"scripts": {
1717
"build": "node build",
18-
"test": "jest --env node --verbose --runInBand --bail",
18+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1919
"test-unit": "pnpm test test/unit.test.ts",
2020
"test-e2e": "pnpm test test/integration-*"
2121
},
@@ -25,6 +25,7 @@
2525
"@types/node": "14.18.33",
2626
"@vercel/build-utils": "6.8.3",
2727
"@vercel/ncc": "0.24.0",
28-
"execa": "^1.0.0"
28+
"execa": "^1.0.0",
29+
"jest-junit": "16.0.0"
2930
}
3031
}

‎packages/redwood/package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"scripts": {
1616
"build": "node build.js",
1717
"test-e2e": "pnpm test test/test.js",
18-
"test": "jest --env node --verbose --bail --runInBand",
18+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand",
1919
"test-unit": "pnpm test test/prepare-cache.test.js"
2020
},
2121
"dependencies": {
@@ -29,6 +29,7 @@
2929
"@types/semver": "6.0.0",
3030
"@vercel/build-utils": "6.8.3",
3131
"execa": "3.2.0",
32-
"fs-extra": "11.1.0"
32+
"fs-extra": "11.1.0",
33+
"jest-junit": "16.0.0"
3334
}
3435
}

‎packages/remix/package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
"scripts": {
1313
"build": "node build.js",
14-
"test": "jest --env node --verbose --bail --runInBand",
14+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand",
1515
"test-unit": "pnpm test test/unit.*test.*",
1616
"test-e2e": "pnpm test test/integration.test.ts"
1717
},
@@ -31,6 +31,7 @@
3131
"@remix-run/dev": "npm:@vercel/[email protected]",
3232
"@types/jest": "27.5.1",
3333
"@types/node": "14.18.33",
34-
"@types/semver": "7.3.13"
34+
"@types/semver": "7.3.13",
35+
"jest-junit": "16.0.0"
3536
}
3637
}

‎packages/routing-utils/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"license": "Apache-2.0",
1616
"scripts": {
1717
"build": "tsc",
18-
"test": "jest --env node --verbose --runInBand --bail",
18+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
1919
"test-unit": "pnpm test"
2020
},
2121
"dependencies": {
@@ -25,6 +25,7 @@
2525
"@types/jest": "27.4.1",
2626
"@types/node": "14.18.33",
2727
"ajv": "^6.0.0",
28+
"jest-junit": "16.0.0",
2829
"typescript": "4.9.5"
2930
},
3031
"optionalDependencies": {

‎packages/ruby/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
},
1717
"scripts": {
1818
"build": "node build",
19-
"test": "jest --env node --verbose --runInBand --bail",
19+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail",
2020
"test-e2e": "pnpm test"
2121
},
2222
"devDependencies": {
@@ -26,6 +26,7 @@
2626
"@vercel/ncc": "0.24.0",
2727
"execa": "2.0.4",
2828
"fs-extra": "^7.0.1",
29+
"jest-junit": "16.0.0",
2930
"semver": "6.1.1"
3031
}
3132
}

‎packages/static-build/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
},
1515
"scripts": {
1616
"build": "node build",
17-
"test": "jest --env node --verbose --bail --runInBand",
17+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --bail --runInBand",
1818
"test-unit": "pnpm test test/build.test.ts test/gatsby.test.ts test/prepare-cache.test.ts",
1919
"test-e2e": "pnpm test test/integration-*.test.js"
2020
},
@@ -43,6 +43,7 @@
4343
"fs-extra": "10.0.0",
4444
"get-port": "5.0.0",
4545
"is-port-reachable": "2.0.1",
46+
"jest-junit": "16.0.0",
4647
"ms": "2.1.2",
4748
"node-fetch": "2.6.7",
4849
"rc9": "1.2.0",

‎packages/static-config/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"scripts": {
1212
"build": "tsc",
1313
"test-unit": "pnpm test",
14-
"test": "jest --env node --verbose --runInBand --bail"
14+
"test": "jest --reporters=default --reporters=jest-junit --env node --verbose --runInBand --bail"
1515
},
1616
"files": [
1717
"dist"
@@ -25,6 +25,7 @@
2525
"@swc/core": "1.2.182",
2626
"@types/jest": "27.4.1",
2727
"@types/node": "14.18.33",
28+
"jest-junit": "16.0.0",
2829
"typescript": "4.9.5"
2930
}
3031
}

0 commit comments

Comments
 (0)