3.0.9 #415
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: wipac ci/cd | |
| on: [push] | |
| env: | |
| CI_TEST: true | |
| TEST_JSON_DIR: tests/resources | |
| jobs: | |
| py-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.versions.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: versions | |
| uses: WIPACrepo/[email protected] | |
| ############################################################################# | |
| # LINTERS | |
| ############################################################################# | |
| flake8: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| - uses: WIPACrepo/[email protected] | |
| mypy: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/[email protected] | |
| ############################################################################# | |
| # PACKAGING | |
| ############################################################################# | |
| writable-branch-detect: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| OKAY: ${{ steps.detect.outputs.OKAY }} | |
| steps: | |
| - name: is this a non-dependabot branch? | |
| id: detect | |
| # dependabot can't access normal secrets | |
| # & don't run non-branch triggers (like tags) | |
| # & we don't want to trigger an update on PR's merge to main/master/default (which is a branch) | |
| run: | | |
| if [[ \ | |
| ${{github.actor}} != 'dependabot[bot]' && \ | |
| ${{github.ref_type}} == 'branch' && \ | |
| ${{format('refs/heads/{0}', github.event.repository.default_branch)}} != ${{github.ref}} \ | |
| ]]; then | |
| echo "OKAY=true" >> "$GITHUB_OUTPUT" | |
| echo "yes, this branch is compatible" | |
| else | |
| echo "OKAY=false" >> "$GITHUB_OUTPUT" | |
| echo "no, this branch is incompatible" | |
| fi | |
| py-setup: | |
| needs: [ writable-branch-detect ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: WIPACrepo/[email protected] | |
| with: | |
| base-keywords: WIPAC IceCube | |
| py-dependencies: | |
| needs: [ writable-branch-detect ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: needs.writable-branch-detect.outputs.OKAY == 'true' | |
| uses: WIPACrepo/[email protected] | |
| ############################################################################# | |
| # TESTS | |
| ############################################################################# | |
| unit-tests: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: run | |
| run: | | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[tests] | |
| export OIDC_CLIENT_SECRETS=$(realpath resources/dummy_client_secrets_for_web_app.json) | |
| pytest -vvv tests/unit | |
| test-build-docker: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| file: Dockerfile | |
| tags: moudash:local | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| integration_test_ingest_type: [ | |
| xlsx, | |
| mongodump_v2, | |
| mongodump_v3, | |
| ] | |
| services: | |
| mongo: | |
| image: mongo:3 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| cache-from: type=gha | |
| # cache-to: type=gha,mode=min | |
| file: Dockerfile | |
| tags: moudash:local | |
| load: true | |
| - name: mongo prep | |
| run: | | |
| sudo apt update | |
| sudo apt install wget curl gnupg2 software-properties-common apt-transport-https ca-certificates lsb-release | |
| curl -fsSL https://www.mongodb.org/static/pgp/server-6.0.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/mongodb-6.gpg | |
| echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list | |
| sudo apt update | |
| sudo apt install mongodb-org mongodb-database-tools | |
| set -x | |
| # ingest data | |
| if [ "${{ matrix.integration_test_ingest_type }}" = "mongodump_v2" ]; then | |
| if [[ ! $(grep -o 'LIVE_COLLECTION' $TEST_JSON_DIR/v2-mo-supplemental.json) ]]; then | |
| exit 1 # the ingested collection must be LIVE_COLLECTION | |
| fi | |
| mongoimport -d mo-supplemental -c LIVE_COLLECTION --type json --file $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo -c LIVE_COLLECTION --jsonArray --type json --file $TEST_JSON_DIR/v2-mo.json | |
| # -> v3 includes v2 data | |
| elif [ "${{ matrix.integration_test_ingest_type }}" = "mongodump_v3" ]; then | |
| sed -i 's/LIVE_COLLECTION/100/g' $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo-supplemental -c 100 --type json --file $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo -c 100 --type json --jsonArray --file $TEST_JSON_DIR/v2-mo.json | |
| if [[ ! $(grep -o 'LIVE_COLLECTION' $TEST_JSON_DIR/v3-mo-supplemental.json) ]]; then | |
| exit 1 # the ingested collection must be LIVE_COLLECTION | |
| fi | |
| mongoimport -d mo-supplemental -c LIVE_COLLECTION --type json --file $TEST_JSON_DIR/v3-mo-supplemental.json | |
| mongoimport -d mo -c LIVE_COLLECTION --jsonArray --type json --file $TEST_JSON_DIR/v3-mo.json | |
| fi | |
| - name: run | |
| run: | | |
| # rest server (background) | |
| docker run --network="host" --rm -i --name rest \ | |
| --env CI_TEST=true \ | |
| moudash:local \ | |
| python -m rest_server --override-krs-insts ./resources/dummy-krs-data.json \ | |
| & | |
| sleep 30 | |
| # make test script | |
| DIR="test-script-dir" | |
| mkdir $DIR | |
| echo "#!/bin/bash" >> $DIR/test-script.sh | |
| echo "set -xe" >> $DIR/test-script.sh | |
| echo "pip install .[tests]" >> $DIR/test-script.sh | |
| echo "python -m pytest -vvv tests/integration --exitfirst" >> $DIR/test-script.sh | |
| chmod +x $DIR/test-script.sh | |
| cat $DIR/test-script.sh | |
| # test | |
| docker run --network="host" --rm -i --name test \ | |
| --env CI_TEST=true \ | |
| --env OIDC_CLIENT_SECRETS=resources/dummy_client_secrets_for_web_app.json \ | |
| --env INTEGRATION_TEST_INGEST_TYPE=${{ matrix.integration_test_ingest_type }} \ | |
| --mount type=bind,source=$(realpath $DIR),target=/local/$DIR \ | |
| moudash:local \ | |
| /local/$DIR/test-script.sh | |
| - name: mongodump | |
| if: always() | |
| run: | | |
| set -x | |
| out_dir="./mongodump-jsons" | |
| tmp_file="temp.js" | |
| echo "print('_ ' + db.getCollectionNames())" > $tmp_file | |
| for database in mo mo-supplemental; do | |
| echo "dumping $database..." | |
| for col in $(mongosh $database $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '); do | |
| mongoexport -d $database -c $col -o "$out_dir/exp_${database}_${col}.json" | |
| done | |
| done | |
| more $out_dir/* | cat | |
| - name: dump rest logs | |
| if: always() | |
| run: | | |
| docker logs rest || true | |
| - name: dump mongo logs | |
| if: always() | |
| run: | | |
| docker logs "${{ job.services.mongo.id }}" || true | |
| ############################################################################# | |
| # GITHUB RELEASE | |
| ############################################################################# | |
| release: | |
| # only run on main/master/default | |
| if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref | |
| needs: [flake8, mypy, py-setup, py-dependencies, unit-tests, integration-tests] | |
| runs-on: ubuntu-latest | |
| concurrency: release | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - name: Python Semantic Release | |
| uses: python-semantic-release/[email protected] | |
| with: | |
| github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| # repository_username: __token__ | |
| # repository_password: ${{ secrets.PYPI_TOKEN }} |