ci(global): INFRA-902 test metrics delivery (part 1) (#5849)

SavelevMatthew · web-flow · commit d8c5c0ad13ae · 2025-02-27T17:06:14.000+05:00
* ci(global): INFRA-902 test metrics delivery * fix(global): INFRA-902 replace fetch with KS one * fix(global): INFRA-902 replace fetch with node-fetch * fix(global): INFRA-902 txt * fix(global): INFRA-902 status * ci(global): INFRA-902 test prom-client * ci(global): INFRA-902 test prom-client * ci(global): INFRA-902 test prometheus-remote-write * ci(global): INFRA-902 add custom tests reporter * ci(global): INFRA-902 TMP CHANGE TRIGGER * ci(global): INFRA-902 add envs to CI jobs * ci(global): INFRA-902 change jest configs to include reporter * Revert "ci(global): INFRA-902 TMP CHANGE TRIGGER" This reverts commit ec349a3.
diff --git a/.github/workflows/nodejs.condo.ci.yml b/.github/workflows/nodejs.condo.ci.yml
@@ -313,10 +313,20 @@ jobs:
           docker exec condo-container yarn workspace @app/address-service makemigrations --check
       - name: Run apps and tests
         run: |
-          docker exec condo-container sh -c "(\
+          docker exec \
+          -e PROMETHEUS_RW_SERVER_URL=$PROMETHEUS_RW_SERVER_URL \
+          -e PROMETHEUS_USER=$PROMETHEUS_USER \
+          -e PROMETHEUS_PASSWORD=$PROMETHEUS_PASSWORD \
+          -e GH_REF_NAME=$GH_REF_NAME \
+          condo-container sh -c "(\
           yarn workspace @app/address-service start & \
           node bin/wait-apps-apis.js -f address-service) && \
           yarn workspace @app/address-service test"
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
 
   run-condo-tests:
     name: Condo Tests
@@ -389,7 +399,17 @@ jobs:
           docker exec condo-container sh -c "yarn workspace @app/condo start" 2>&1 > ./test_logs/dev_portal_api.condo.dev.log &
           docker exec condo-container sh -c "yarn workspace @app/dev-portal-api start" 2>&1 > ./test_logs/dev_portal_api.dev-portal-api.dev.log &
           docker exec condo-container sh -c "node bin/wait-apps-apis.js -f condo dev-portal-api"
-          docker exec condo-container sh -c "yarn workspace @app/dev-portal-api test" 2>&1 > ./test_logs/dev_portal_api.tests.log
+          docker exec \
+          -e PROMETHEUS_RW_SERVER_URL=$PROMETHEUS_RW_SERVER_URL \
+          -e PROMETHEUS_USER=$PROMETHEUS_USER \
+          -e PROMETHEUS_PASSWORD=$PROMETHEUS_PASSWORD \
+          -e GH_REF_NAME=$GH_REF_NAME \
+          condo-container sh -c "yarn workspace @app/dev-portal-api test" 2>&1 > ./test_logs/dev_portal_api.tests.log
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
       - name: Collect docker logs on failure
         if: failure()
         uses: jwalton/gh-docker-logs@v1
@@ -427,6 +447,11 @@ jobs:
           yarn install --immutable
       - name: Run @open-condo/migrator tests
         run: yarn workspace @open-condo/migrator test
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
 
   run-rb-tests:
     name: RB Tests
@@ -464,11 +489,21 @@ jobs:
           docker exec condo-container yarn workspace @app/rb makemigrations --check
       - name: Run apps and tests
         run: |
-          docker exec condo-container sh -c "(\
+          docker exec \
+          -e PROMETHEUS_RW_SERVER_URL=$PROMETHEUS_RW_SERVER_URL \
+          -e PROMETHEUS_USER=$PROMETHEUS_USER \
+          -e PROMETHEUS_PASSWORD=$PROMETHEUS_PASSWORD \
+          -e GH_REF_NAME=$GH_REF_NAME \
+          condo-container sh -c "(\
           yarn workspace @app/condo start & \
           yarn workspace @app/rb start & \
           node bin/wait-apps-apis.js -f condo rb) && \
           yarn workspace @app/rb test"
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
 
   run-registry-tests:
     name: Registry-importer Tests
@@ -506,11 +541,21 @@ jobs:
           docker exec condo-container yarn workspace @app/registry-importer makemigrations --check
       - name: Run apps and tests
         run: |
-          docker exec condo-container sh -c "(\
+          docker exec \
+          -e PROMETHEUS_RW_SERVER_URL=$PROMETHEUS_RW_SERVER_URL \
+          -e PROMETHEUS_USER=$PROMETHEUS_USER \
+          -e PROMETHEUS_PASSWORD=$PROMETHEUS_PASSWORD \
+          -e GH_REF_NAME=$GH_REF_NAME \
+          condo-container sh -c "(\
           yarn workspace @app/condo start & \
           yarn workspace @app/registry-importer start & \
           node bin/wait-apps-apis.js -f condo registry-importer) && \
           yarn workspace @app/registry-importer test"
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
 
   run-webhooks-tests:
     name: Webhooks Tests
@@ -532,4 +577,9 @@ jobs:
           npm i -g turbo
           yarn install --immutable
       - name: Test webhooks utils
-        run: yarn workspace @open-condo/webhooks jest
+        run: yarn workspace @open-condo/webhooks test
+        env:
+          PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
+          PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
+          PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
+          GH_REF_NAME: ${{ github.ref_name }}
diff --git a/apps/address-service/jest.config.js b/apps/address-service/jest.config.js
@@ -1,12 +1,14 @@
 /** @type {import('jest').Config} */
 module.exports = {
+    rootDir: './',
+    reporters: ['default', '<rootDir>/../../bin/report-failed-tests.js'],
     projects: [
         {
             testRunner: 'jasmine2',
             displayName: 'schema',
             testEnvironment: 'node',
-            testMatch: [`${__dirname}/schema/**/*.test.js`, `${__dirname}/domains/**/schema/*.test.js`],
-            setupFilesAfterEnv: [`${__dirname}/jest.setupTest.js`],
+            testMatch: ['<rootDir>/schema/**/*.test.js', '<rootDir>/domains/**/schema/*.test.js'],
+            setupFilesAfterEnv: ['<rootDir>/jest.setupTest.js'],
             // NOTE: need to pass uuid export syntax through babel
             transformIgnorePatterns: ['/node_modules/(?!(uuid)/)'],
         },
@@ -19,7 +21,7 @@ module.exports = {
             transform: {
                 '\\.[jt]sx?$': 'babel-jest',
             },
-            setupFilesAfterEnv: [`${__dirname}/jest.setupSpec.js`],
+            setupFilesAfterEnv: ['<rootDir>/jest.setupSpec.js'],
             // NOTE: need to pass uuid export syntax through babel
             transformIgnorePatterns: ['/node_modules/(?!(uuid)/)'],
         },
diff --git a/apps/dev-portal-api/jest.config.js b/apps/dev-portal-api/jest.config.js
@@ -1,20 +1,23 @@
+/** @type {import('jest').Config} */
 module.exports = {
+    rootDir: './',
+    reporters: ['default', '<rootDir>/../../bin/report-failed-tests.js'],
     projects: [
         {
             testRunner: 'jasmine2',
             displayName: 'schema tests',
             testEnvironment: 'node',
-            testMatch: [`${__dirname}/schema/**/*.test.js`, `${__dirname}/domains/**/schema/*.test.js`],
-            setupFilesAfterEnv: [`${__dirname}/jest.setupTest.js`],
+            testMatch: ['<rootDir>/schema/**/*.test.js', '<rootDir>/domains/**/schema/*.test.js'],
+            setupFilesAfterEnv: ['<rootDir>/jest.setupTest.js'],
         },
         {
             testRunner: 'jasmine2',
             displayName: 'schema specs',
             testEnvironment: 'node',
             testMatch: [
-                `${__dirname}/domains/**/schema/**/*.spec.js`,
+                '<rootDir>/domains/**/schema/**/*.spec.js',
             ],
-            setupFilesAfterEnv: [`${__dirname}/jest.setupSpec.js`],
+            setupFilesAfterEnv: ['<rootDir>/jest.setupSpec.js'],
         },
     ],
 }
diff --git a/bin/report-failed-tests.js b/bin/report-failed-tests.js
@@ -0,0 +1,109 @@
+const path = require('path')
+
+const fetch = require('node-fetch')
+const { pushTimeseries } = require('prometheus-remote-write')
+
+const REPO_ROOT = path.join(__dirname, '..')
+const PROMETHEUS_REMOTE_WRITE_URL = process.env.PROMETHEUS_RW_SERVER_URL
+const PROMETHEUS_AUTH_USER = process.env.PROMETHEUS_USER
+const PROMETHEUS_AUTH_PASSWORD = process.env.PROMETHEUS_PASSWORD
+const RUNTIME_REF_NAME = process.env.GH_REF_NAME || 'local'
+const BATCH_SIZE = 10
+
+/** @implements {import('@jest/reporters').Reporter} */
+class PrometheusReporter {
+    #config = null
+
+    constructor () {
+        if (PROMETHEUS_REMOTE_WRITE_URL && PROMETHEUS_AUTH_USER && PROMETHEUS_AUTH_PASSWORD) {
+            this.#config = {
+                url: PROMETHEUS_REMOTE_WRITE_URL,
+                auth: {
+                    username: PROMETHEUS_AUTH_USER,
+                    password: PROMETHEUS_AUTH_PASSWORD,
+                },
+                fetch,
+                labels: {
+                    __name__: 'ci_failed_test',
+                    ref: RUNTIME_REF_NAME,
+                },
+            }
+        }
+    }
+
+    /**
+     * @param {string} testName
+     * @param {string} testPath
+     * @param {number} timestamp
+     */
+    async #reportFailedTest ({ testName, testPath, timestamp }) {
+        if (this.#config) {
+            await pushTimeseries({
+                labels: {
+                    test_name: testName,
+                    test_file: testPath,
+                },
+                samples: [
+                    { value: 1, timestamp },
+                ],
+            }, this.#config)
+        }
+    }
+
+    /**
+     * @param {Set<import('@jest/reporters').TestContext>} testContexts
+     * @param {import('@jest/reporters').AggregatedResult} runResults
+     */
+    async onRunComplete (
+        testContexts,
+        runResults
+    ) {
+
+        /**
+         * @type {Array<{ testName: string, testPath: string, timestamp: number }>}
+         */
+        const failedTests = []
+
+        if (runResults.numFailedTests === 0) {
+            return
+        }
+
+        for (const testFileResult of runResults.testResults) {
+            if (testFileResult.numFailingTests === 0) {
+                continue
+            }
+
+            const absoluteTestFilePath = testFileResult.testFilePath
+            const testFilePath = path.relative(REPO_ROOT, absoluteTestFilePath)
+            const fileEndTimestamp = testFileResult.perfStats.end
+
+            for (const assertionResult of testFileResult.testResults) {
+                if (assertionResult.status === 'failed') {
+                    const testNameParts = [...assertionResult.ancestorTitles, assertionResult.title]
+                    // Clean tabulation / spaces and join
+                    const testFullName = testNameParts
+                        .map(part => part.replace(/\s+/g, ' '))
+                        .join(' > ')
+                    failedTests.push({
+                        testName: testFullName,
+                        testPath: testFilePath,
+                        timestamp: fileEndTimestamp,
+                    })
+                }
+            }
+        }
+
+        for (let startIdx = 0; startIdx < failedTests.length; startIdx += BATCH_SIZE) {
+            const endIdx = Math.min(startIdx + BATCH_SIZE, failedTests.length)
+            const batch = failedTests.slice(startIdx, endIdx)
+
+            await Promise.allSettled(batch.map(test => this.#reportFailedTest({
+                testName: test.testName,
+                testPath: test.testPath,
+                timestamp: test.timestamp,
+            })))
+        }
+    }
+}
+
+module.exports = PrometheusReporter
diff --git a/package.json b/package.json
@@ -29,6 +29,7 @@
     "@faker-js/faker": "^7.6.0",
     "@graphql-codegen/cli": "1.21.8",
     "@graphql-codegen/typescript": "1.23.0",
+    "@jest/reporters": "^29.7.0",
     "@mono-pub/commit-analyzer": "^1.0.5",
     "@mono-pub/git": "^1.0.5",
     "@mono-pub/github": "^1.0.5",
@@ -54,6 +55,7 @@
     "node-fetch": "^2.6.7",
     "postcss": "^8.3.5",
     "postcss-less": "^6.0.0",
+    "prometheus-remote-write": "^0.5.0",
     "stylelint": "^14.11.0",
     "stylelint-config-rational-order": "^0.1.2",
     "stylelint-config-standard": "^28.0.0",
diff --git a/packages/migrator/jest.config.js b/packages/migrator/jest.config.js
@@ -3,6 +3,7 @@ module.exports = {
     preset: 'ts-jest',
     testEnvironment: 'node',
     rootDir: './',
+    reporters: ['default', '<rootDir>/../../bin/report-failed-tests.js'],
     testPathIgnorePatterns: ['/node_modules/'],
     moduleFileExtensions: ['js', 'ts'],
     projects: [
diff --git a/packages/webhooks/jest.config.js b/packages/webhooks/jest.config.js
@@ -1,12 +1,15 @@
+/** @type {import('jest').Config} */
 module.exports = {
+    rootDir: './',
+    reporters: ['default', '<rootDir>/../../bin/report-failed-tests.js'],
     projects: [
         {
             displayName: 'Specs',
             testEnvironment: 'node',
             testRunner: 'jasmine2',
-            testMatch: [`${__dirname}/**/*.spec.js`],
+            testMatch: ['<rootDir>/**/*.spec.js'],
             testPathIgnorePatterns: ['/tasks/'],
-            setupFilesAfterEnv: [`${__dirname}/jest.setupSpec.js`],
+            setupFilesAfterEnv: ['<rootDir>/jest.setupSpec.js'],
         },
     ],
 }
diff --git a/packages/webhooks/package.json b/packages/webhooks/package.json
@@ -7,6 +7,9 @@
     "keystone",
     "webhooks"
   ],
+  "scripts": {
+    "test": "jest"
+  },
   "version": "0.0.0-development",
   "repository": {
     "type": "git",
diff --git a/yarn.lock b/yarn.lock
@@ -45608,6 +45608,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"prometheus-remote-write@npm:^0.5.0":
+  version: 0.5.0
+  resolution: "prometheus-remote-write@npm:0.5.0"
+  dependencies:
+    protobufjs: ^7.2.4
+    snappyjs: ^0.6.1
+  peerDependencies:
+    node-fetch: ^2.6.7
+  peerDependenciesMeta:
+    node-fetch:
+      optional: true
+  checksum: e0fed2f5dcdda9b1cf1eab111c12342a1b1ba6e1c6970ce92a27e6f27185b6e94dfdabcaeb322d51ad01ab5c75a992be648f6ebac1a4ce3610e524474aa031ad
+  languageName: node
+  linkType: hard
+
 "promise-inflight@npm:^1.0.1":
   version: 1.0.1
   resolution: "promise-inflight@npm:1.0.1"
@@ -45738,7 +45753,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"protobufjs@npm:^7.2.5, protobufjs@npm:^7.2.6, protobufjs@npm:^7.3.2":
+"protobufjs@npm:^7.2.4, protobufjs@npm:^7.2.5, protobufjs@npm:^7.2.6, protobufjs@npm:^7.3.2":
   version: 7.4.0
   resolution: "protobufjs@npm:7.4.0"
   dependencies:
@@ -50250,6 +50265,7 @@ __metadata:
     "@faker-js/faker": ^7.6.0
     "@graphql-codegen/cli": 1.21.8
     "@graphql-codegen/typescript": 1.23.0
+    "@jest/reporters": ^29.7.0
     "@mono-pub/commit-analyzer": ^1.0.5
     "@mono-pub/git": ^1.0.5
     "@mono-pub/github": ^1.0.5
@@ -50281,6 +50297,7 @@ __metadata:
     node-fetch: ^2.6.7
     postcss: ^8.3.5
     postcss-less: ^6.0.0
+    prometheus-remote-write: ^0.5.0
     stylelint: ^14.11.0
     stylelint-config-rational-order: ^0.1.2
     stylelint-config-standard: ^28.0.0
@@ -51414,6 +51431,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"snappyjs@npm:^0.6.1":
+  version: 0.6.1
+  resolution: "snappyjs@npm:0.6.1"
+  checksum: ff1e129207e8dead5674fdf066c4b91ea8aa8f4a09b25a8de14904dce6c2a361463a0d502f6d2c074529cc3fe0aa176ed1f990528ed8f324f23b542ebc14caff
+  languageName: node
+  linkType: hard
+
 "socket.io-adapter@npm:~2.5.2":
   version: 2.5.2
   resolution: "socket.io-adapter@npm:2.5.2"
