From ca735ca10df148deec8be838e166d252060b318d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 4 Apr 2017 15:36:20 -0700 Subject: [PATCH] PYTHON-1234 Evergreen - add CPython coverage reporting. --- .coveragerc | 11 +++-- .evergreen/config.yml | 104 ++++++++++++++++++++++++++++++++++------ .evergreen/run-tests.sh | 23 +++++++-- 3 files changed, 116 insertions(+), 22 deletions(-) diff --git a/.coveragerc b/.coveragerc index 57a0186ee..9fbe05c09 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,11 +1,14 @@ [run] branch = True -omit = pymongo/ssl*.py,pymongo/auth.py,test/* source = pymongo,bson,gridfs +[paths] +source= + . + /data/mci/*/src + [report] exclude_lines = - if (.*and +)*PY3( and.*)*: if (.*and +)*_use_c( and.*)*: def has_c def get_version_string @@ -16,5 +19,7 @@ exclude_lines = _use_c = True if __name__ == '__main__': partial_branches = - if (.*and +)*not PY3( and.*)*: if (.*and +)*not _use_c( and.*)*: + +[html] +directory = htmlcov diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 997336c01..dede6d6ab 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -190,23 +190,60 @@ functions: display_name: "Rendered docs" "upload coverage" : - - command: shell.exec - params: - silent: true - script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: ${PROJECT_DIRECTORY}/coverage/index.html - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html + local_file: ${PROJECT_DIRECTORY}/.coverage + optional: true + # Upload the coverage report for all tasks in a single build to the same directory. + remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name} bucket: mciuploads permissions: public-read content_type: text/html - display_name: "Coverage Report" + display_name: "Raw Coverage Report" + + "download and merge coverage" : + - command: shell.exec + params: + silent: true + working_dir: "src" + script: | + export AWS_ACCESS_KEY_ID=${aws_key} + export AWS_SECRET_ACCESS_KEY=${aws_secret} + + # Download all the task coverage files. + aws s3 cp --recursive s3://mciuploads/${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/coverage/ coverage/ + - command: shell.exec + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + # Coverage combine merges (and removes) all the coverage files and + # generates a new .coverage file in the current directory. + ls -la coverage/ + /opt/python/3.6/bin/python3 -m coverage combine coverage/coverage.* + /opt/python/3.6/bin/python3 -m coverage html -d htmlcov + # Upload the resulting html coverage report. + - command: shell.exec + params: + silent: true + working_dir: "src" + script: | + export AWS_ACCESS_KEY_ID=${aws_key} + export AWS_SECRET_ACCESS_KEY=${aws_secret} + aws s3 cp htmlcov/ s3://mciuploads/${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/htmlcov/ --recursive --acl public-read --region us-east-1 + # Attach the index.html with s3.put so it shows up in the Evergreen UI. + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/htmlcov/index.html + remote_file: ${UPLOAD_BUCKET}/coverage/${revision}/${version_id}/htmlcov/index.html + bucket: mciuploads + permissions: public-read + content_type: text/html + display_name: "Coverage Report HTML" "upload scan artifacts" : - command: shell.exec @@ -370,7 +407,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} GREEN_FRAMEWORK=${GREEN_FRAMEWORK} C_EXTENSIONS=${C_EXTENSIONS} AUTH=${AUTH} SSL=${SSL} sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + PYTHON_BINARY=${PYTHON_BINARY} GREEN_FRAMEWORK=${GREEN_FRAMEWORK} C_EXTENSIONS=${C_EXTENSIONS} COVERAGE=${COVERAGE} AUTH=${AUTH} SSL=${SSL} sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh "run enterprise auth tests": - command: shell.exec @@ -448,6 +485,7 @@ pre: post: # Disabled, causing timeouts # - func: "upload working dir" + - func: "upload coverage" - func: "upload mo artifacts" - func: "upload test results" - func: "stop mongo-orchestration" @@ -707,6 +745,27 @@ tasks: PYTHON_BINARY: /opt/python/2.7/bin/python # }}} + - name: "coverage-report" + tags: ["coverage"] + depends_on: + # BUILD-3165: We can't use "*" (all tasks) and specify "variant". + # Instead list out all coverage tasks using tags. + - name: ".standalone" + variant: ".coverage_tag" + # Run the coverage task even if some tasks fail. + status: "*" + # Run the coverage task even if some tasks are not scheduled in a patch build. + patch_optional: true + - name: ".replica_set" + variant: ".coverage_tag" + status: "*" + patch_optional: true + - name: ".sharded_cluster" + variant: ".coverage_tag" + status: "*" + patch_optional: true + commands: + - func: "download and merge coverage" axes: @@ -992,6 +1051,14 @@ axes: display_name: "Python 3.6" variables: PYTHON_BINARY: "/cygdrive/c/python/Python36/python.exe" + - id: coverage + display_name: "Coverage" + values: + - id: "coverage" + display_name: "Coverage" + tags: ["coverage_tag"] + variables: + COVERAGE: "coverage" buildvariants: @@ -1052,14 +1119,15 @@ buildvariants: - ".3.4" - matrix_name: "tests-python-version-ubuntu1204-test-ssl" - matrix_spec: {"python-version": "*", auth: "*", ssl: "*"} + matrix_spec: {"python-version": "*", auth: "*", ssl: "*", coverage: "*"} exclude_spec: # PYTHON-498: disable Jython SSL tests python-version: "jython2.7" # EVG-1410: exlcude_spec must specifiy values for all axes auth: "*" ssl: "ssl" - display_name: "${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl}" + coverage: "*" + display_name: "${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl} ${coverage}" run_on: ubuntu1204-test tasks: - ".latest" @@ -1070,14 +1138,15 @@ buildvariants: - ".2.4" - matrix_name: "tests-python-version-ubuntu1204-without-c-extensions" - matrix_spec: {"python-version": "*", "c-extensions": "without-c-extensions", auth: "noauth", ssl: "nossl"} + matrix_spec: {"python-version": "*", "c-extensions": "without-c-extensions", auth: "noauth", ssl: "nossl", coverage: "*"} exclude_spec: # These interpreters are always tested without extensions. - python-version: ["pypy", "pypy3", "jython2.7"] c-extensions: "*" auth: "*" ssl: "*" - display_name: "${c-extensions} ${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl}" + coverage: "*" + display_name: "${c-extensions} ${python-version} Ubuntu 12.04 (x86_64) ${auth} ${ssl} ${coverage}" run_on: ubuntu1204-test tasks: - ".latest" @@ -1232,6 +1301,11 @@ buildvariants: tasks: - name: "no-server" +- name: "Coverage Report" + display_name: "Coverage Report" + run_on: ubuntu1604-test + tasks: + - name: "coverage-report" # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 026132606..dfc84a31b 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -6,8 +6,9 @@ set -o errexit # Exit the script with error if any of the commands fail # AUTH Set to enable authentication. Defaults to "noauth" # SSL Set to enable SSL. Defaults to "nossl" # PYTHON_BINARY The Python version to use. Defaults to whatever is available -# GREEN_FRAMEWORK The green framwork to test with, if any. +# GREEN_FRAMEWORK The green framework to test with, if any. # C_EXTENSIONS Pass --no_ext to setup.py, or not. +# COVERAGE If non-empty, run the test suite with coverage. AUTH=${AUTH:-noauth} @@ -15,6 +16,7 @@ SSL=${SSL:-nossl} PYTHON_BINARY=${PYTHON_BINARY:-} GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-} C_EXTENSIONS=${C_EXTENSIONS:-} +COVERAGE=${COVERAGE:-} export JAVA_HOME=/opt/java/jdk8 @@ -54,6 +56,21 @@ $PYTHON -c 'import sys; print(sys.version)' # Run the tests, and store the results in Evergreen compatible XUnit XML # files in the xunit-results/ directory. +# Run the tests with coverage if requested and coverage is installed. +# Only cover CPython. Jython and PyPy report suspiciously low coverage. +COVERAGE_OR_PYTHON="$PYTHON" +COVERAGE_ARGS="" +if [ -n "$COVERAGE" -a $PYTHON_IMPL = "CPython" ]; then + COVERAGE_BIN="$(dirname "$PYTHON")/coverage" + if $COVERAGE_BIN --version; then + echo "INFO: coverage is installed, running tests with coverage..." + COVERAGE_OR_PYTHON="$COVERAGE_BIN" + COVERAGE_ARGS="run --branch" + else + echo "INFO: coverage is not installed, running tests without coverage..." + fi +fi + $PYTHON setup.py clean if [ -z "$GREEN_FRAMEWORK" ]; then if [ -z "$C_EXTENSIONS" -a $PYTHON_IMPL = "CPython" ]; then @@ -66,10 +83,8 @@ if [ -z "$GREEN_FRAMEWORK" ]; then # This will set a non-zero exit status if either import fails, # causing this script to exit. $PYTHON -c "from bson import _cbson; from pymongo import _cmessage" - $PYTHON setup.py test $OUTPUT - else - $PYTHON setup.py $C_EXTENSIONS test $OUTPUT fi + $COVERAGE_OR_PYTHON $COVERAGE_ARGS setup.py $C_EXTENSIONS test $OUTPUT else # --no_ext has to come before "test" so there is no way to toggle extensions here. $PYTHON green_framework_test.py $GREEN_FRAMEWORK $OUTPUT