From 5d6d8ca68e7cae6b0a39057c5941225958ed8543 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 9 Aug 2023 14:13:59 -0500 Subject: [PATCH] PYTHON-3885 Use tox in remaining Evergreen Tests (#1347) --- .evergreen/config.yml | 13 +++-- .evergreen/run-doctests.sh | 7 --- .evergreen/run-enterprise-auth-tests.sh | 32 ---------- .evergreen/run-mongodb-aws-ecs-test.sh | 20 +++---- .evergreen/run-mongodb-aws-test.sh | 35 ++--------- .evergreen/run-mongodb-oidc-test.sh | 40 ++----------- .evergreen/run-ocsp-tests.sh | 25 -------- .evergreen/run-perf-tests.sh | 21 ++----- .evergreen/run-tests.sh | 78 +++++++++++++++++++++++-- .github/workflows/test-python.yml | 8 +-- CONTRIBUTING.rst | 12 ++-- README.rst | 11 ++-- RELEASE.rst | 4 -- doc/installation.rst | 2 +- test/__init__.py | 4 ++ 15 files changed, 123 insertions(+), 189 deletions(-) delete mode 100644 .evergreen/run-doctests.sh delete mode 100644 .evergreen/run-enterprise-auth-tests.sh delete mode 100644 .evergreen/run-ocsp-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 48d376b0b..ffe666443 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -370,7 +370,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh + PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test "run tests": # If testing FLE, start the KMS mock servers, first create the virtualenv. @@ -477,7 +477,6 @@ functions: fi if [ -n "${test_loadbalancer}" ]; then export TEST_LOADBALANCER=1 - export LOAD_BALANCER=1 export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" fi @@ -537,8 +536,11 @@ functions: set +x . ./prepare_enterprise_auth.sh rm -f ./prepare_enterprise_auth.sh - - PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-enterprise-auth-tests.sh + PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ + PYTHON_BINARY="${PYTHON_BINARY}" \ + TEST_ENTERPRISE_AUTH=1 \ + AUTH=auth \ + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run atlas tests": - command: shell.exec @@ -964,10 +966,11 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + TEST_OCSP=1 \ PYTHON_BINARY=${PYTHON_BINARY} \ CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg run-valid-ocsp-server: - command: shell.exec diff --git a/.evergreen/run-doctests.sh b/.evergreen/run-doctests.sh deleted file mode 100644 index be71f1789..000000000 --- a/.evergreen/run-doctests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -${PYTHON_BINARY} -m pip install tox -${PYTHON_BINARY} -m tox -m doc-test diff --git a/.evergreen/run-enterprise-auth-tests.sh b/.evergreen/run-enterprise-auth-tests.sh deleted file mode 100644 index e86e489d0..000000000 --- a/.evergreen/run-enterprise-auth-tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Don't trace to avoid secrets showing up in the logs -set -o errexit - -echo "Running enterprise authentication tests" - -export DB_USER="bob" -export DB_PASSWORD="pwd123" -if [ "Windows_NT" = "$OS" ]; then - echo "Setting GSSAPI_PASS" - export GSSAPI_PASS=${SASL_PASS} - export GSSAPI_CANONICALIZE="true" -else - # BUILD-3830 - touch ${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty - export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty - - echo "Writing keytab" - echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab - echo "Running kinit" - kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} -fi -echo "Setting GSSAPI variables" -export GSSAPI_HOST=${SASL_HOST} -export GSSAPI_PORT=${SASL_PORT} -export GSSAPI_PRINCIPAL=${PRINCIPAL} - - -echo "Running tests" -${PYTHON_BINARY} setup.py clean -${PYTHON_BINARY} setup.py test --xunit-output=xunit-results diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index fcadea208..e604774d2 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -22,16 +22,12 @@ set -o xtrace # Install python3.7 with pip. apt-get update -apt install python3.7 python3-pip -y +apt-get install python3.7 python3-pip build-essential python3.7-dev -y -authtest () { - echo "Running MONGODB-AWS ECS authentication tests with $PYTHON" - $PYTHON --version - $PYTHON -m pip install --upgrade wheel setuptools pip - cd src - $PYTHON -m pip install '.[aws]' - $PYTHON test/auth_aws/test_auth_aws.py -v - cd - -} - -PYTHON="python3.7" authtest +export PYTHON_BINARY="python3.7" +export TEST_AUTH_AWS=1 +export AUTH="auth" +export SET_XTRACE_ON=1 +cd src +python3.7 -m pip install --user tox +bash .evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index b2a4fd146..772dd367f 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -39,37 +39,12 @@ fi # show test output set -x -# Workaround macOS python 3.9 incompatibility with system virtualenv. -if [ "$(uname -s)" = "Darwin" ]; then - VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/python3 -m virtualenv" -else - VIRTUALENV=$(command -v virtualenv) -fi - -authtest () { - if [ "Windows_NT" = "$OS" ]; then - PYTHON=$(cygpath -m $PYTHON) - fi - - echo "Running MONGODB-AWS authentication tests with $PYTHON" - $PYTHON --version - - $VIRTUALENV -p $PYTHON --never-download venvaws - if [ "Windows_NT" = "$OS" ]; then - . venvaws/Scripts/activate - else - . venvaws/bin/activate - fi - python -m pip install '.[aws]' - python test/auth_aws/test_auth_aws.py -v - deactivate - rm -rf venvaws -} - -PYTHON=${PYTHON_BINARY:-} -if [ -z "$PYTHON" ]; then +if [ -z "$PYTHON_BINARY" ]; then echo "Cannot test without specifying PYTHON_BINARY" exit 1 fi -authtest +export TEST_AUTH_AWS=1 +export AUTH="auth" +export SET_XTRACE_ON=1 +bash ./.evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 46bb77957..e84df2539 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -45,41 +45,13 @@ if [ "$ASSERT_NO_URI_CREDS" = "true" ]; then fi fi -# show test output -set -x - -# Workaround macOS python 3.9 incompatibility with system virtualenv. -if [ "$(uname -s)" = "Darwin" ]; then - VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/python3 -m virtualenv" -else - VIRTUALENV=$(command -v virtualenv) -fi - -authtest () { - if [ "Windows_NT" = "$OS" ]; then - PYTHON=$(cygpath -m $PYTHON) - fi - - echo "Running MONGODB-OIDC authentication tests with $PYTHON" - $PYTHON --version - - $VIRTUALENV -p $PYTHON --never-download venvoidc - if [ "Windows_NT" = "$OS" ]; then - . venvoidc/Scripts/activate - else - . venvoidc/bin/activate - fi - python -m pip install -U pip setuptools - python -m pip install '.[aws]' - python test/auth_aws/test_auth_oidc.py -v - deactivate - rm -rf venvoidc -} - -PYTHON=${PYTHON_BINARY:-} -if [ -z "$PYTHON" ]; then +if [ -z "$PYTHON_BINARY" ]; then echo "Cannot test without specifying PYTHON_BINARY" exit 1 fi -authtest +export TEST_AUTH_OIDC=1 +export AUTH_MECH="SCRAM-SHA-256" +export AUTH="auth" +export SET_XTRACE_ON=1 +bash ./.evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-ocsp-tests.sh b/.evergreen/run-ocsp-tests.sh deleted file mode 100644 index 0eb101aaa..000000000 --- a/.evergreen/run-ocsp-tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -# For createvirtualenv. -. .evergreen/utils.sh - -if [ -z "$PYTHON_BINARY" ]; then - echo "No python binary specified" - PYTHON=$(command -v python3) || true - if [ -z "$PYTHON" ]; then - echo "Cannot test without python3 installed!" - exit 1 - fi -else - PYTHON="$PYTHON_BINARY" -fi - -createvirtualenv $PYTHON ocsptest -trap "deactivate; rm -rf ocsptest" EXIT HUP - -python -m pip install --prefer-binary pyopenssl requests service_identity - -OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED} CA_FILE=${CA_FILE} python test/ocsp/test_ocsp.py diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index bc447a956..91d4b72a7 100644 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -13,21 +13,8 @@ cd .. export TEST_PATH="${PROJECT_DIRECTORY}/driver-performance-test-data" export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" -MTCBIN=/opt/mongodbtoolchain/v3/bin -VIRTUALENV="$MTCBIN/virtualenv -p $MTCBIN/python3" +export PYTHON_BINARY=/opt/mongodbtoolchain/v3/bin/python3 +export C_EXTENSIONS=1 +export PERF_TEST=1 -$VIRTUALENV pyperftest -. pyperftest/bin/activate -python -m pip install simplejson - -python setup.py build_ext -i -start_time=$(date +%s) -python test/performance/perf_test.py --locals -end_time=$(date +%s) -elapsed_secs=$((end_time-start_time)) - -cat results.json - -echo "{\"failures\": 0, \"results\": [{\"status\": \"pass\", \"exit_code\": 0, \"test_file\": \"BenchMarkTests\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json - -cat report.json +bash ./.evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 1e7fd9b4f..d3a24f628 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -20,6 +20,11 @@ set -o errexit # Exit the script with error if any of the commands fail # TEST_FLE_AZURE_AUTO If non-empy, test auto FLE on Azure # TEST_FLE_GCP_AUTO If non-empy, test auto FLE on GCP # TEST_PYOPENSSL If non-empy, test with PyOpenSSL +# TEST_ENTERPRISE_AUTH If non-empty, test with Enterprise Auth +# TEST_AUTH_AWS If non-empty, test AWS Auth Mechanism +# TEST_AUTH_OIDC If non-empty, test OIDC Auth Mechanism +# TEST_PERF If non-empty, run performance tests +# TEST_OCSP If non-empty, run OCSP tests # TEST_ENCRYPTION_PYOPENSSL If non-empy, test encryption with PyOpenSSL if [ -n "${SET_XTRACE_ON}" ]; then @@ -48,6 +53,34 @@ if [ "$AUTH" != "noauth" ]; then fi fi +if [ -n "$TEST_ENTERPRISE_AUTH" ]; then + if [ "Windows_NT" = "$OS" ]; then + echo "Setting GSSAPI_PASS" + export GSSAPI_PASS=${SASL_PASS} + export GSSAPI_CANONICALIZE="true" + else + # BUILD-3830 + touch krb5.conf.empty + export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty + + echo "Writing keytab" + echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab + echo "Running kinit" + kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} + fi + echo "Setting GSSAPI variables" + export GSSAPI_HOST=${SASL_HOST} + export GSSAPI_PORT=${SASL_PORT} + export GSSAPI_PRINCIPAL=${PRINCIPAL} +fi + +if [ -n "$TEST_LOADBALANCER" ]; then + export LOAD_BALANCER=1 + export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}" + export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}" + export TEST_ARGS="test/test_load_balancer.py" +fi + if [ "$SSL" != "nossl" ]; then export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" @@ -59,15 +92,15 @@ if [ "$SSL" != "nossl" ]; then fi if [ "$COMPRESSORS" = "snappy" ]; then - pip install '.[snappy]' + python -m pip install '.[snappy]' PYTHON=python elif [ "$COMPRESSORS" = "zstd" ]; then - pip install zstandard + python -m pip install zstandard fi # PyOpenSSL test setup. if [ -n "$TEST_PYOPENSSL" ]; then - pip install '.[ocsp]' + python -m pip install '.[ocsp]' fi if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then @@ -75,14 +108,14 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE # Work around for root certifi not being installed. # TODO: Remove after PYTHON-3827 if [ "$(uname -s)" = "Darwin" ]; then - pip install certifi + python -m pip install certifi CERT_PATH=$(python -c "import certifi; print(certifi.where())") export SSL_CERT_FILE=${CERT_PATH} export REQUESTS_CA_BUNDLE=${CERT_PATH} export AWS_CA_BUNDLE=${CERT_PATH} fi - pip install '.[encryption]' + python -m pip install '.[encryption]' if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin # PYTHON-2808 Ensure this machine has the CA cert for google KMS. @@ -128,7 +161,7 @@ fi if [ -n "$TEST_ENCRYPTION" ]; then if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then - pip install '.[ocsp]' + python -m pip install '.[ocsp]' fi # Get access to the AWS temporary credentials: @@ -172,6 +205,27 @@ if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then TEST_ARGS="test/test_data_lake.py" fi +if [ -n "$TEST_OCSP" ]; then + python -m pip install ".[ocsp]" + TEST_ARGS="test/ocsp/test_ocsp.py" +fi + +if [ -n "$TEST_AUTH_AWS" ]; then + python -m pip install ".[aws]" + TEST_ARGS="test/auth_aws/test_auth_aws.py" +fi + +if [ -n "$TEST_AUTH_OIDC" ]; then + python -m pip install ".[aws]" + TEST_ARGS="test/auth_aws/test_auth_oidc.py" +fi + +if [ -n "$PERF_TEST" ]; then + python -m pip install simplejson + start_time=$(date +%s) + TEST_ARGS="test/performance/perf_test.py" +fi + echo "Running $AUTH tests over $SSL with python $PYTHON" python -c 'import sys; print(sys.version)' @@ -199,3 +253,15 @@ else python -m pip install $GREEN_FRAMEWORK python green_framework_test.py $GREEN_FRAMEWORK fi + +# Handle perf test post actions. +if [ -n "$PERF_TEST" ]; then + end_time=$(date +%s) + elapsed_secs=$((end_time-start_time)) + + cat results.json + + echo "{\"failures\": 0, \"results\": [{\"status\": \"pass\", \"exit_code\": 0, \"test_file\": \"BenchMarkTests\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json + + cat report.json +fi diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 3d25aaaa8..10e0240ec 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -32,7 +32,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | pip install tox @@ -54,7 +54,7 @@ jobs: with: python-version: "3.8" cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | pip install tox @@ -80,7 +80,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | pip install tox @@ -102,7 +102,7 @@ jobs: - uses: actions/setup-python@v2 with: cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | pip install tox diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 87a94934d..964b90d34 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -34,7 +34,7 @@ General Guidelines - Avoid backward breaking changes if at all possible. - Write inline documentation for new classes and methods. - Write tests and make sure they pass (make sure you have a mongod - running on the default port, then execute ``python setup.py test`` + running on the default port, then execute ``tox -m test`` from the cmd line to run the test suite). - Add yourself to doc/contributors.rst :) @@ -73,11 +73,13 @@ button. Running Tests Locally --------------------- - Ensure you have started the appropriate Mongo Server(s). -- Run ``python setup.py test`` to run all of the tests. -- Run ``python setup.py test -s test...`` to +- Run ``pip install tox`` to use ``tox`` for testing or run ``pip install -e ".[test]"`` to run ``pytest`` directly. +- Run ``tox -m test`` or ``pytest`` to run all of the tests. +- Append ``test/.py::::`` to run specific tests. You can omit the ```` to test a full class and the ```` to test a full module. For example: - ``python setup.py test -s test.test_change_stream.TestUnifiedChangeStreamsErrors.test_change_stream_errors_on_ElectionInProgress``. + ``tox -m test test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress``. +- Use the ``-k`` argument to select tests by pattern. Running Load Balancer Tests Locally ----------------------------------- @@ -85,7 +87,7 @@ Running Load Balancer Tests Locally - Clone ``drivers-evergreen-tools``: ``git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git``. - Start the servers using ``LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh``. - Start the load balancer using: ``MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start``. -- Run the tests from the ``pymongo`` checkout directory using: ``LOADBALANCER=1 TEST_LOADBALANCER=1 SINGLE_MONGOS_LB_URI='mongodb://127.0.0.1:8000/?loadBalanced=true' MULTI_MONGOS_LB_URI='mongodb://127.0.0.1:8001/?loadBalanced=true' MONGODB_URI='mongodb://localhost:27017,localhost:27018/' python setup.py test -s test.test_load_balancer``. +- Run the tests from the ``pymongo`` checkout directory using: ``TEST_LOADBALANCER=1 tox -m test-eg``. Re-sync Spec Tests ------------------ diff --git a/README.rst b/README.rst index 6274e2c9d..3172ecb8a 100644 --- a/README.rst +++ b/README.rst @@ -109,7 +109,6 @@ MONGODB-AWS authentication requires `pymongo-auth-aws $ python -m pip install "pymongo[aws]" - OCSP (Online Certificate Status Protocol) requires `PyOpenSSL `_, `requests `_, `service_identity @@ -142,7 +141,7 @@ command:: Additional dependencies are: -- (to generate documentation) sphinx_ +- (to generate documentation or run tests) tox_ Examples ======== @@ -187,9 +186,7 @@ Documentation Documentation is available at `pymongo.readthedocs.io `_. -To build the documentation, you will need to install sphinx_. -Documentation can be generated by running **python -setup.py doc**. Generated documentation can be found in the +Documentation can be generated by running **tox -m doc**. Generated documentation can be found in the *doc/build/html/* directory. Learning Resources @@ -201,7 +198,7 @@ MongoDB Learn - `Python courses