add condition coverage

Author: tomato42

.travis.yml

@@ -6,8 +6,11 @@ language: python
 cache: pip
 before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
+# place the slowest (instrumental and py2.6) first
 matrix:
   include:
+      - python: 2.7
+        env: INSTRUMENTAL=yes
       - python: 2.6
         env: TOX_ENV=py26
       - python: 2.7
@@ -29,13 +32,48 @@ matrix:
       - python: pypy3
         env: TOX_ENV=pypy3
 
+# for instrumental we're checking if the coverage changed from base branch
+# so collect that info
+before_install:
+  - |
+       echo -e "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
+               "TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG\n" \
+               "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
+               "TRAVIS_COMMIT=$TRAVIS_COMMIT\n" \
+               "TRAVIS_PYTHON_VERSION=$TRAVIS_PYTHON_VERSION"
+  - |
+       # workaround https://github.com/travis-ci/travis-ci/issues/2666
+       if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+         URL="https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST}.patch"
+         # `--location` makes curl follow redirects
+         PR_FIRST=$(curl --silent --show-error --location $URL | head -1 | grep -o -E '\b[0-9a-f]{40}\b' | tr -d '\n')
+         TRAVIS_COMMIT_RANGE=$PR_FIRST^..$TRAVIS_COMMIT
+       fi
+  # sanity check current commit
+  - git rev-parse HEAD
+  - echo "TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE"
+  - git fetch origin master:refs/remotes/origin/master
+
+
 install:
   - pip list
   - if [[ -e build-requirements-${TRAVIS_PYTHON_VERSION}.txt ]]; then travis_retry pip install -r build-requirements-${TRAVIS_PYTHON_VERSION}.txt; else travis_retry pip install -r build-requirements.txt; fi
+  - if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi
   - pip list
 script:
-  - tox -e $TOX_ENV
+  - if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi
   - tox -e speed
+  - |
+      if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
+        git checkout $PR_FIRST^
+        instrumental -t ecdsa -i 'test.*|_version' python -m pytest src
+        instrumental -f .instrumental.cov -s
+        instrumental -f .instrumental.cov -s | python diff-instrumental.py --save .diff-instrumental
+        git checkout $TRAVIS_COMMIT
+        instrumental -t ecdsa -i 'test.*|_version' python -m pytest src
+        instrumental -f .instrumental.cov -sr
+        instrumental -f .instrumental.cov -s | python diff-instrumental.py --read .diff-instrumental --fail-under 70 --max-difference 0.1
+      fi
 after_success:
   - coveralls
 

diff-instrumental.py

@@ -0,0 +1,62 @@
+import sys
+import getopt
+
+fail_under = None
+max_difference = 0
+read_location = None
+save_location = None
+
+argv = sys.argv[1:]
+
+opts, args = getopt.getopt(
+    argv, "s:r:",
+    ["fail-under=", "max-difference=", "save=", "read="])
+if args:
+    raise ValueError("Unexpected parameters: {0}".format(args))
+for opt, arg in opts:
+    if opt == "-s" or opt == "--save":
+        save_location = arg
+    elif opt == "-r" or opt == "--read":
+        read_location = arg
+    elif opt == "--fail-under":
+        fail_under = float(arg)/100.0
+    elif opt == "--max-difference":
+        max_difference = float(arg)/100.0
+    else:
+        raise ValueError("Unknown option: {0}".format(opt))
+
+total_hits = 0
+total_count = 0
+
+for line in sys.stdin.readlines():
+    if not line.startswith("ecdsa"):
+        continue
+
+    fields = line.split()
+    hit, count = fields[1].split('/')
+    total_hits += int(hit)
+    total_count += int(count)
+
+coverage = total_hits * 1.0 / total_count
+
+if read_location:
+    with open(read_location, "r") as f:
+        old_coverage = float(f.read())
+    print("Old coverage: {0:6.2f}%".format(old_coverage*100))
+
+if save_location:
+    with open(save_location, "w") as f:
+        f.write("{0:1.40f}".format(coverage))
+
+print("Coverage: {0:6.2f}%".format(coverage*100))
+
+if read_location:
+    print("Difference: {0:6.2f}%".format((old_coverage - coverage)*100))
+
+if fail_under and coverage < fail_under:
+    print("ERROR: Insufficient coverage.", file=sys.stderr)
+    sys.exit(1)
+
+if read_location and coverage - old_coverage < max_difference:
+    print("ERROR: Too big decrease in coverage", file=sys.stderr)
+    sys.exit(1)