PYTHON-1234 Evergreen - add CPython coverage reporting.

2017-04-04 15:36:20 -07:00 · 2017-04-04 15:36:20 -07:00 · ca735ca10d
commit ca735ca10d
parent 6aaa1f71aa
3 changed files with 116 additions and 22 deletions
--- a/.coveragerc
+++ b/.coveragerc
@ -1,11 +1,14 @@
 [run]
 branch = True
-omit = pymongo/ssl*.py,pymongo/auth.py,test/*
 source = pymongo,bson,gridfs

+[paths]
+source=
+    .
+    /data/mci/*/src
+
 [report]
 exclude_lines =
-    if (.*and +)*PY3( and.*)*:
    if (.*and +)*_use_c( and.*)*:
    def has_c
    def get_version_string
@ -16,5 +19,7 @@ exclude_lines =
    _use_c = True
    if __name__ == '__main__':
 partial_branches =
-    if (.*and +)*not PY3( and.*)*:
    if (.*and +)*not _use_c( and.*)*:
+
+[html]
+directory = htmlcov
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -190,23 +190,60 @@ functions:
        display_name: "Rendered docs"

  "upload coverage" :
-    - command: shell.exec
-      params:
-        silent: true
-        script: |
-           export AWS_ACCESS_KEY_ID=${aws_key}
-           export AWS_SECRET_ACCESS_KEY=${aws_secret}
-           aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
-        local_file:  ${PROJECT_DIRECTORY}/coverage/index.html
-        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html
+        local_file:  ${PROJECT_DIRECTORY}/.coverage
+        optional: true
+        # Upload the coverage report for all tasks in a single build to the same directory.
+        remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
        bucket: mciuploads
        permissions: public-read
        content_type: text/html
-        display_name: "Coverage Report"
+        display_name: "Raw Coverage Report"
+
+  "download and merge coverage" :
+    - command: shell.exec
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+          export AWS_ACCESS_KEY_ID=${aws_key}
+          export AWS_SECRET_ACCESS_KEY=${aws_secret}
+
+          # Download all the task coverage files.
+          aws s3 cp --recursive s3://mciuploads/${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/coverage/ coverage/
+    - command: shell.exec
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          # Coverage combine merges (and removes) all the coverage files and
+          # generates a new .coverage file in the current directory.
+          ls -la coverage/
+          /opt/python/3.6/bin/python3 -m coverage combine coverage/coverage.*
+          /opt/python/3.6/bin/python3 -m coverage html -d htmlcov
+    # Upload the resulting html coverage report.
+    - command: shell.exec
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+           export AWS_ACCESS_KEY_ID=${aws_key}
+           export AWS_SECRET_ACCESS_KEY=${aws_secret}
+           aws s3 cp htmlcov/ s3://mciuploads/${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/htmlcov/ --recursive --acl public-read --region us-east-1
+    # Attach the index.html with s3.put so it shows up in the Evergreen UI.
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file:  ${PROJECT_DIRECTORY}/htmlcov/index.html
+        remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/htmlcov/index.html
+        bucket: mciuploads
+        permissions: public-read
+        content_type: text/html
+        display_name: "Coverage Report HTML"

  "upload scan artifacts" :
    - command: shell.exec
@ -370,7 +407,7 @@ functions:
        working_dir: "src"
        script: |
          ${PREPARE_SHELL}
-          PYTHON_BINARY=${PYTHON_BINARY} GREEN_FRAMEWORK=${GREEN_FRAMEWORK} C_EXTENSIONS=${C_EXTENSIONS} AUTH=${AUTH} SSL=${SSL} sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+          PYTHON_BINARY=${PYTHON_BINARY} GREEN_FRAMEWORK=${GREEN_FRAMEWORK} C_EXTENSIONS=${C_EXTENSIONS} COVERAGE=${COVERAGE} AUTH=${AUTH} SSL=${SSL} sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh

  "run enterprise auth tests":
    - command: shell.exec
@ -448,6 +485,7 @@ pre:
 post:
  # Disabled, causing timeouts
  # - func: "upload working dir"
+  - func: "upload coverage"
  - func: "upload mo artifacts"
  - func: "upload test results"
  - func: "stop mongo-orchestration"
@ -707,6 +745,27 @@ tasks:
            PYTHON_BINARY: /opt/python/2.7/bin/python

 # }}}
+    - name: "coverage-report"
+      tags: ["coverage"]
+      depends_on:
+        # BUILD-3165: We can't use "*" (all tasks) and specify "variant".
+        # Instead list out all coverage tasks using tags.
+        - name: ".standalone"
+          variant: ".coverage_tag"
+          # Run the coverage task even if some tasks fail.
+          status: "*"
+          # Run the coverage task even if some tasks are not scheduled in a patch build.
+          patch_optional: true
+        - name: ".replica_set"
+          variant: ".coverage_tag"
+          status: "*"
+          patch_optional: true
+        - name: ".sharded_cluster"
+          variant: ".coverage_tag"
+          status: "*"
+          patch_optional: true
+      commands:
+        - func: "download and merge coverage"


 axes:
@ -992,6 +1051,14 @@ axes:
        display_name: "Python 3.6"
        variables:
           PYTHON_BINARY: "/cygdrive/c/python/Python36/python.exe"
+  - id: coverage
+    display_name: "Coverage"
+    values:
+      - id: "coverage"
+        display_name: "Coverage"
+        tags: ["coverage_tag"]
+        variables:
+           COVERAGE: "coverage"


 buildvariants:
@ -1052,14 +1119,15 @@ buildvariants:
     - ".3.4"

 - matrix_name: "tests-python-version-ubuntu1204-test-ssl"
-  matrix_spec: {"python-version": "*", auth: "*", ssl: "*"}
+  matrix_spec: {"python-version": "*", auth: "*", ssl: "*", coverage: "*"}
  exclude_spec:
    # PYTHON-498: disable Jython SSL tests
    python-version: "jython2.7"
    # EVG-1410: exlcude_spec must specifiy values for all axes
    auth: "*"
    ssl: "ssl"
-  display_name: "${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl}"
+    coverage: "*"
+  display_name: "${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl} ${coverage}"
  run_on: ubuntu1204-test
  tasks:
     - ".latest"
@ -1070,14 +1138,15 @@ buildvariants:
     - ".2.4"

 - matrix_name: "tests-python-version-ubuntu1204-without-c-extensions"
-  matrix_spec: {"python-version": "*", "c-extensions": "without-c-extensions", auth: "noauth", ssl: "nossl"}
+  matrix_spec: {"python-version": "*", "c-extensions": "without-c-extensions", auth: "noauth", ssl: "nossl", coverage: "*"}
  exclude_spec:
   # These interpreters are always tested without extensions.
   - python-version: ["pypy", "pypy3", "jython2.7"]
     c-extensions: "*"
     auth: "*"
     ssl: "*"
-  display_name: "${c-extensions} ${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl}"
+     coverage: "*"
+  display_name: "${c-extensions} ${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl} ${coverage}"
  run_on: ubuntu1204-test
  tasks:
     - ".latest"
@ -1232,6 +1301,11 @@ buildvariants:
  tasks:
    - name: "no-server"

+- name: "Coverage Report"
+  display_name: "Coverage Report"
+  run_on: ubuntu1604-test
+  tasks:
+     - name: "coverage-report"

      # Platform notes
      # i386 builds of OpenSSL or Cyrus SASL are not available
--- a/.evergreen/run-tests.sh
+++ b/.evergreen/run-tests.sh
@ -6,8 +6,9 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #       AUTH                    Set to enable authentication. Defaults to "noauth"
 #       SSL                     Set to enable SSL. Defaults to "nossl"
 #       PYTHON_BINARY           The Python version to use. Defaults to whatever is available
-#       GREEN_FRAMEWORK         The green framwork to test with, if any.
+#       GREEN_FRAMEWORK         The green framework to test with, if any.
 #       C_EXTENSIONS            Pass --no_ext to setup.py, or not.
+#       COVERAGE                If non-empty, run the test suite with coverage.


 AUTH=${AUTH:-noauth}
@ -15,6 +16,7 @@ SSL=${SSL:-nossl}
 PYTHON_BINARY=${PYTHON_BINARY:-}
 GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-}
 C_EXTENSIONS=${C_EXTENSIONS:-}
+COVERAGE=${COVERAGE:-}

 export JAVA_HOME=/opt/java/jdk8

@ -54,6 +56,21 @@ $PYTHON -c 'import sys; print(sys.version)'
 # Run the tests, and store the results in Evergreen compatible XUnit XML
 # files in the xunit-results/ directory.

+# Run the tests with coverage if requested and coverage is installed.
+# Only cover CPython. Jython and PyPy report suspiciously low coverage.
+COVERAGE_OR_PYTHON="$PYTHON"
+COVERAGE_ARGS=""
+if [ -n "$COVERAGE" -a $PYTHON_IMPL = "CPython" ]; then
+    COVERAGE_BIN="$(dirname "$PYTHON")/coverage"
+    if $COVERAGE_BIN --version; then
+        echo "INFO: coverage is installed, running tests with coverage..."
+        COVERAGE_OR_PYTHON="$COVERAGE_BIN"
+        COVERAGE_ARGS="run --branch"
+    else
+        echo "INFO: coverage is not installed, running tests without coverage..."
+    fi
+fi
+
 $PYTHON setup.py clean
 if [ -z "$GREEN_FRAMEWORK" ]; then
    if [ -z "$C_EXTENSIONS" -a $PYTHON_IMPL = "CPython" ]; then
@ -66,10 +83,8 @@ if [ -z "$GREEN_FRAMEWORK" ]; then
        # This will set a non-zero exit status if either import fails,
        # causing this script to exit.
        $PYTHON -c "from bson import _cbson; from pymongo import _cmessage"
-        $PYTHON setup.py test $OUTPUT
-    else
-        $PYTHON setup.py $C_EXTENSIONS test $OUTPUT
    fi
+    $COVERAGE_OR_PYTHON $COVERAGE_ARGS setup.py $C_EXTENSIONS test $OUTPUT
 else
    # --no_ext has to come before "test" so there is no way to toggle extensions here.
    $PYTHON green_framework_test.py $GREEN_FRAMEWORK $OUTPUT