add quick mutation testing to travis

Author: tomato42

.travis.yml

@@ -6,9 +6,13 @@ language: python
 cache: pip
 before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
-# place the slowest (instrumental and py2.6) first
+# place the slowest (instrumental, mutation and py2.6) first
 matrix:
   include:
+      - python: 3.7
+        dist: xenial
+        sudo: true
+        env: MUTATION=yes
       - python: 2.7
         env: INSTRUMENTAL=yes
       - python: 2.6
@@ -74,6 +78,13 @@ install:
         travis_retry pip install -r build-requirements.txt;
       fi
   - if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi
+    #- if [[ $MUTATION ]]; then travis_retry pip install cosmic-ray; fi
+  # temporary, until https://github.com/sixty-north/cosmic-ray/pull/491 is merged
+  - if [[ $MUTATION ]]; then pushd $(mktemp -d); fi
+  - if [[ $MUTATION ]]; then git clone --branch sampling --depth 1 https://github.com/tomato42/cosmic-ray.git; fi
+  - if [[ $MUTATION ]]; then cd cosmic-ray; fi
+  - if [[ $MUTATION ]]; then python setup.py install; fi
+  - if [[ $MUTATION ]]; then popd; fi
   - pip list
 script:
   - if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi
@@ -101,6 +112,26 @@ script:
       if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
           instrumental -f .instrumental.cov -s | python diff-instrumental.py --read .diff-instrumental --fail-under 70 --max-difference -0.1
       fi
+  # cosmic-ray (mutation testing) runs
+  - if [[ $MUTATION ]]; then cosmic-ray init cosmic-ray.toml session.sqlite; fi
+  - if [[ $MUTATION ]]; then cosmic-ray baseline --report session.sqlite; fi
+  - if [[ $MUTATION ]]; then cr-report --show-output session.baseline.sqlite; fi
+  - |
+      if [[ $MUTATION ]]; then
+          cosmic-ray exec session.sqlite &
+          COSMIC_PID=$!
+          # make sure that we output something every 10 minutes (otherwise travis will kill us)
+          while kill -s 0 $COSMIC_PID; do
+            sleep 300
+            cr-report session.sqlite | tail -n 3;
+          done &
+          REPORT_PID=$!
+          # kill exec after 25 minutes
+          (sleep $((60*25)); kill $COSMIC_PID) &
+      fi
+  - if [[ $MUTATION ]]; then wait $COSMIC_PID ; kill $REPORT_PID ; true; fi
+  - if [[ $MUTATION ]]; then cr-report --show-output session.sqlite | tail -n 40; fi
+  - if [[ $MUTATION ]]; then cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite; fi
 after_success:
-  - if [[ -z $INSTRUMENTAL ]]; then coveralls; fi
+  - if [[ -z $INSTRUMENTAL && -z $MUTATION ]]; then coveralls; fi
 

cosmic-ray.sh

@@ -7,3 +7,4 @@ cr-report --show-output session.baseline.sqlite
 cosmic-ray exec session.sqlite
 cr-report session.sqlite
 cr-html session.sqlite > session.html
+cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite

cosmic-ray.toml

@@ -1,7 +1,7 @@
 [cosmic-ray]
 module-path = "src"
 python-version = ""
-timeout = 120.0
+timeout = 300.0
 exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*']
 test-command = "pytest -x src/"