Skip to content

Commit 6cfcfea

Browse files
TylerJang27TylerJang27
authored
Setup Flaky Test monitoring (#774)
Adds an upload step so that we upload all our test results. Validated on [webapp](https://app.trunk-staging.io/trunk-staging-org/trunk-io/plugins/test-analytics?statuses=Broken%2CFlaky%2CNew). The next step once the results seem to be reliable would be to turn on test quarantining and run with that action (note that it does not support Windows).
1 parent 22b9e81 commit 6cfcfea

11 files changed

+137
-6
lines changed

‎.github/actions/action_tests/action.yaml

+15-1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ runs:
5454
shell: bash
5555
working-directory: ${{ inputs.path }}
5656
env:
57+
JEST_SUITE_NAME: Action Tests
5758
PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
5859
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
5960

@@ -64,10 +65,23 @@ runs:
6465
breakpoint-id: trunk-plugins-action-tests
6566
shell: bash
6667
working-directory: ${{ inputs.path }}
67-
trunk-token: ${{ inputs.trunk-token }}
68+
trunk-token: ${{ inputs.trunk-token }}
6869
org: trunk-staging-org
6970
run:
7071
npm test ${{ inputs.append-args }} ${{ env.PLATFORM_APPEND_ARGS }} --passWithNoTests --ci
7172
env:
73+
JEST_SUITE_NAME: Action Tests
7274
PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
7375
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
76+
77+
- name: Upload results
78+
# TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
79+
if: "!cancelled() && runner.os == 'Linux'"
80+
uses: trunk-io/analytics-uploader@main
81+
with:
82+
junit-paths: junit.xml
83+
org-slug: trunk-staging-org
84+
token: ${{ inputs.trunk-token }}
85+
continue-on-error: true
86+
env:
87+
TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io

‎.github/actions/linter_tests/action.yaml

+25-3
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,12 @@ inputs:
2424
description: Token to login for sourcery test
2525
required: true
2626
trunk-token:
27-
description: CI debugger api token
27+
description: CI debugger api token (org token)
2828
required: true
29+
ref-type:
30+
description: release or main
31+
required: false
32+
default: main
2933

3034
runs:
3135
# TODO(Tyler): See if this can be converted to a js action
@@ -91,7 +95,10 @@ runs:
9195
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
9296
SOURCERY_TOKEN: ${{ inputs.sourcery-token }}
9397
# Debug recurrent eslint circular JSON errors
94-
DEBUG: Driver:eslint:*,Driver:cspell:*,Driver:nixpkgs-fmt:*
98+
DEBUG: Driver:eslint:*,Driver:nixpkgs-fmt:*
99+
JEST_SUITE_NAME: Linter Tests
100+
JEST_JUNIT_SUITE_NAME:
101+
"{title} ${{ runner.os }} ${{ inputs.ref-type }} ${{ inputs.linter-version }}"
95102

96103
- name: Run plugin tests
97104
if: runner.os != 'Windows'
@@ -101,7 +108,7 @@ runs:
101108
breakpoint-id: trunk-plugins-linter-tests
102109
shell: bash
103110
working-directory: ${{ inputs.path }}
104-
trunk-token: ${{ inputs.trunk-token }}
111+
trunk-token: ${{ inputs.trunk-token }}
105112
org: trunk-staging-org
106113
run: npm test ${{ inputs.append-args }} ${{ env.PLATFORM_APPEND_ARGS }} --ci --runInBand
107114
env:
@@ -110,3 +117,18 @@ runs:
110117
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
111118
SOURCERY_TOKEN: ${{ inputs.sourcery-token }}
112119
DEBUG: Driver:nixpkgs-fmt:*, Driver:eslint:*
120+
JEST_SUITE_NAME: Linter Tests
121+
JEST_JUNIT_SUITE_NAME:
122+
"{title} ${{ runner.os }} ${{ inputs.ref-type }} ${{ inputs.linter-version }}"
123+
124+
- name: Upload results
125+
# TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
126+
if: "!cancelled() && runner.os == 'Linux'"
127+
uses: trunk-io/analytics-uploader@main
128+
with:
129+
junit-paths: junit.xml
130+
org-slug: trunk-staging-org
131+
token: ${{ inputs.trunk-token }}
132+
continue-on-error: true
133+
env:
134+
TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io

‎.github/actions/tool_tests/action.yaml

+21-1
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,12 @@ inputs:
1717
required: false
1818
default: tools --
1919
trunk-token:
20-
description: CI debugger api token
20+
description: CI debugger api token (org token)
2121
required: true
22+
ref-type:
23+
description: release or main
24+
required: false
25+
default: main
2226

2327
runs:
2428
# TODO(Tyler): See if this can be converted to a js action
@@ -56,6 +60,8 @@ runs:
5660
env:
5761
PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
5862
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
63+
JEST_SUITE_NAME: Tool Tests
64+
JEST_JUNIT_SUITE_NAME: "{title} ${{ runner.os }} ${{ inputs.ref-type }}"
5965

6066
- name: Run plugin tests
6167
if: runner.os != 'Windows'
@@ -71,3 +77,17 @@ runs:
7177
env:
7278
PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
7379
PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
80+
JEST_SUITE_NAME: Tool Tests
81+
JEST_JUNIT_SUITE_NAME: "{title} ${{ runner.os }} ${{ inputs.ref-type }}"
82+
83+
- name: Upload results
84+
# TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
85+
if: "!cancelled() && runner.os == 'Linux'"
86+
uses: trunk-io/analytics-uploader@main
87+
with:
88+
junit-paths: junit.xml
89+
org-slug: trunk-staging-org
90+
token: ${{ inputs.trunk-token }}
91+
continue-on-error: true
92+
env:
93+
TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io

‎.github/workflows/nightly.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ jobs:
6666
uses: ./.github/actions/linter_tests
6767
with:
6868
linter-version: ${{ matrix.linter-version }}
69+
ref-type: main
6970
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
7071
trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
7172

@@ -173,6 +174,7 @@ jobs:
173174
with:
174175
linter-version: ${{ matrix.linter-version }}
175176
append-args: linters -- --json --outputFile=${{ matrix.results-file }}-res.json
177+
ref-type: release
176178
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
177179
trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
178180

‎.github/workflows/pr.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ jobs:
150150
uses: ./.github/actions/linter_tests
151151
with:
152152
linter-version: KnownGoodVersion
153+
ref-type: main
153154
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
154155
append-args:
155156
${{ needs.detect_changes.outputs.all-linters }} ${{
@@ -164,6 +165,7 @@ jobs:
164165
uses: ./.github/actions/linter_tests
165166
with:
166167
linter-version: Latest
168+
ref-type: main
167169
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
168170
append-args: ${{ needs.detect_changes.outputs.linters-files }}
169171
trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
@@ -255,6 +257,7 @@ jobs:
255257
uses: ./.github/actions/linter_tests
256258
with:
257259
linter-version: Latest
260+
ref-type: main
258261
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
259262
cli-path: ${{ github.workspace }}\trunk.ps1
260263
append-args: ${{needs.detect_changes.outputs.linters-files }} -- --maxWorkers=5

‎.github/workflows/repo_tests.reusable.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,17 @@ jobs:
3838
- name: Run repo tests
3939
run: npm test tests/repo_tests --ci
4040
env:
41+
JEST_SUITE_NAME: Repo Tests
4142
PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
4243
PLUGINS_TEST_CLI_PATH: ${{ inputs.cli-path }}
44+
45+
- name: Upload results
46+
if: "!cancelled()"
47+
uses: trunk-io/analytics-uploader@main
48+
with:
49+
junit-paths: junit.xml
50+
org-slug: trunk-staging-org
51+
token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
52+
continue-on-error: true
53+
env:
54+
TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io

‎.github/workflows/upload_results.reusable.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ jobs:
255255
uses: ./.github/actions/linter_tests
256256
with:
257257
linter-version: Latest
258+
ref-type: main
258259
append-args: ${{ needs.upload_test_results.outputs.reruns }} -- -u
259260
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
260261
trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}

‎.github/workflows/windows_nightly.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ jobs:
4444
uses: ./.github/actions/linter_tests
4545
with:
4646
linter-version: ${{ matrix.linter-version }}
47+
ref-type: main
4748
sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
4849
cli-path: ${{ github.workspace }}\trunk.ps1
4950
# manually specify more parallelism to avoid bottlenecks

‎jest.config.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
"preset": "ts-jest",
33
"modulePaths": ["<rootDir>"],
44
"setupFilesAfterEnv": ["<rootDir>/tests/jest_setup.ts"],
5-
"reporters": ["<rootDir>/tests/reporter/index.js", "default"],
5+
"reporters": ["<rootDir>/tests/reporter/index.js", "default", "jest-junit"],
66
"testPathIgnorePatterns": ["<rootDir>/node_modules/", "<rootDir>/repo-tools/"]
77
}

‎package-lock.json

+55
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"eslint-plugin-simple-import-sort": "^12.1.0",
3131
"fast-sort": "^3.2.0",
3232
"jest": "^29.3.1",
33+
"jest-junit": "^16.0.0",
3334
"jest-specific-snapshot": "^8.0.0",
3435
"semver": "^7.6.0",
3536
"simple-git": "^3.24.0",

0 commit comments

Comments
 (0)