Parse sitrep from pytest json report

Author: aybchan

.github/eks-workflow-files/transformer-engine/scripts/unittest.sh

@@ -7,13 +7,20 @@
           export TE_PATH=${SRC_PATH_TRANSFORMER_ENGINE}
 
           # 1 GPU per worker, 6 workers per GPU
-          pytest-xdist.sh 1 6 pytest-report-L0-unittest.jsonl bash ${TE_PATH}/qa/L0_jax_unittest/test.sh | tee -a ${LOG_DIR}/tests.log
+          pytest-xdist.sh 1 6 pytest-report-L0-unittest.jsonl bash ${TE_PATH}/qa/L0_jax_unittest/test.sh | tee -a ${LOG_DIR}/pytest_stdout.log
 
           # 8 GPUs per worker, 1 worker per GPU. pytest-xdist.sh allows aggregation
           # into a single .jsonl file of results from multiple pytest invocations
           # inside the test.sh script, so it's useful even with a single worker per
           # device.
-          pytest-xdist.sh 8 1 pytest-report-L0-distributed-unittest.jsonl bash ${TE_PATH}/qa/L0_jax_distributed_unittest/test.sh | tee -a ${LOG_DIR}/tests.log
-          pytest-xdist.sh 8 1 pytest-report-L1-distributed-unittest.jsonl bash ${TE_PATH}/qa/L1_jax_distributed_unittest/test.sh | tee -a ${LOG_DIR}/tests.log
-          
-          touch /opt/output/done
+          pytest-xdist.sh 8 1 pytest-report-L0-distributed-unittest.jsonl bash ${TE_PATH}/qa/L0_jax_distributed_unittest/test.sh | tee -a ${LOG_DIR}/pytest_stdout.log
+          pytest-xdist.sh 8 1 pytest-report-L1-distributed-unittest.jsonl bash ${TE_PATH}/qa/L1_jax_distributed_unittest/test.sh | tee -a ${LOG_DIR}/pytest_stdout.log
+
+          # merge the log files
+          cat \
+            pytest-report-L0-unittest.jsonl \
+            pytest-report-L0-distributed-unittest.jsonl \
+            pytest-report-L1-distributed-unittest.jsonl \
+            > ${LOG_DIR}/pytest-report.jsonl
+
+          touch ${LOG_DIR}/done

.github/eks-workflow-files/transformer-engine/test.yml

@@ -36,8 +36,7 @@ SCRIPT
           done
 
           # Upload logs to S3 bucket
-          cat /opt/output/tests.log
-          aws s3 cp /opt/output/ s3://${S3_BUCKET}/${CI_NAME}/${RUN_ID}/ --recursive --exclude "*" --include "*.log"
+          aws s3 cp /opt/output/ s3://${S3_BUCKET}/${CI_NAME}/${RUN_ID}/ --recursive 
         env:
         - name: RUN_ID
           value: JOB_NAME

.github/workflows/_transformer_engine_eks.yaml

@@ -86,21 +86,21 @@ jobs:
       id: s3-logs-process
       run: |
         LOCAL_DIR=${{ inputs.CI_NAME }}-output
+        REPORT_JSON=pytest-report.jsonl
 
         mkdir -p $LOCAL_DIR
-        # aws s3 cp s3://${{ inputs.S3_BUCKET }}/${{ inputs.CI_NAME }}/${{ env.RUN_NAME }}/summary.txt $LOCAL_DIR/
-        aws s3 cp s3://${{ inputs.S3_BUCKET }}/${{ inputs.CI_NAME }}/${{ env.RUN_NAME }}/ $LOCAL_DIR/ --recursive --exclude "*" --include "*.log"
+        aws s3 cp s3://${{ inputs.S3_BUCKET }}/${{ inputs.CI_NAME }}/${{ env.RUN_NAME }}/ $LOCAL_DIR/ --recursive 
 
-        passed=$(cat $LOCAL_DIR/tests.log | grep 'PASSED opt/transformer' | wc -l || true)
-        failed=$(cat $LOCAL_DIR/tests.log | grep 'FAILED opt/transformer' | wc -l || true)
+        passed_tests=$(cat $LOCAL_DIR/$REPORT_JSON |  jq --slurp | jq '.[] | select(.outcome == "passed") | .outcome' | wc -l || true)
+        failed_tests=$(cat $LOCAL_DIR/$REPORT_JSON |  jq --slurp | jq '.[] | select(.outcome == "failed") | .outcome' | wc -l || true)
 
-        total=$((failed + passed))
-        echo "Passed tests: $passed"
-        echo "Failed tests: $failed"
-        echo "Total tests: $total"
-        echo "PASSED_TESTS=$passed" >> $GITHUB_OUTPUT
-        echo "FAILED_TESTS=$failed" >> $GITHUB_OUTPUT
-        echo "TOTAL_TESTS=$total" >> $GITHUB_OUTPUT
+        total_test=$((failed_tests + passed_tests))
+        echo "Passed tests: $passed_tests"
+        echo "Failed tests: $failed_tests"
+        echo "Total tests: $total_tests"
+        echo "PASSED_TESTS=$passed_tests" >> $GITHUB_OUTPUT
+        echo "FAILED_TESTS=$failed_tests" >> $GITHUB_OUTPUT
+        echo "TOTAL_TESTS=$total_tests" >> $GITHUB_OUTPUT
 
     - name: Generate sitrep
       id: sitrep