From 188edeb21c05ab023f4954d4ced7aae4e0ada67b Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 16 Dec 2020 15:32:25 -1000 Subject: [PATCH] PYTHON-2366 Test OCSP+FLE with Python 3.9 (#534) PYTHON-2449 Move all pypy cryptography/pyopenssl testing to Debian 9.2 with OpenSSL 1.1.0f PYTHON-2449 Fix Windows cryptography installation by upgrading pip and using --prefer-binary (cherry picked from commit 3ecd9479d47363afa78f0f0699b8154f2f1c6f6f) --- .evergreen/config.yml | 51 ++++++++++++++++----- .evergreen/run-ocsp-tests.sh | 34 ++------------ .evergreen/run-tests.sh | 34 ++------------ .evergreen/test-encryption-requirements.txt | 4 ++ .evergreen/utils.sh | 11 +++-- 5 files changed, 61 insertions(+), 73 deletions(-) create mode 100644 .evergreen/test-encryption-requirements.txt diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5e2ce14d3..8000bea13 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1561,6 +1561,7 @@ axes: run_on: debian92-test batchtime: 10080 # 7 days variables: + python3_binary: "/opt/python/3.8/bin/python3" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian92/master/latest/libmongocrypt.tar.gz - id: macos-1014 display_name: "macOS 10.14" @@ -2160,14 +2161,14 @@ buildvariants: - matrix_name: "tests-pyopenssl" matrix_spec: platform: ubuntu-16.04 - python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3.5"] + python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9"] auth: "*" ssl: "ssl" pyopenssl: "*" # Only test "noauth" with Python 3.7. exclude_spec: platform: ubuntu-16.04 - python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9", "pypy", "pypy3.5"] + python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9"] auth: "noauth" ssl: "ssl" pyopenssl: "*" @@ -2177,6 +2178,19 @@ buildvariants: # Test standalone and sharded only on 4.4. - '.4.4' +- matrix_name: "tests-pyopenssl-pypy" + matrix_spec: + platform: debian92 + python-version: ["pypy", "pypy3.5", "pypy3.6"] + auth: "auth" + ssl: "ssl" + pyopenssl: "*" + display_name: "PyOpenSSL ${platform} ${python-version} ${auth}" + tasks: + - '.replica_set !.2.6 !.3.0 !.3.2 !.3.4' + # Test standalone and sharded only on 4.4. + - '.4.4' + - matrix_name: "test-pyopenssl-old-py27" matrix_spec: platform: @@ -2214,7 +2228,7 @@ buildvariants: matrix_spec: platform: rhel62 # RHEL 6.2 does not support Python 3.7.x and later. - python-version: ["2.7", "3.4", "3.5", "3.6", "pypy", "pypy3.5", "pypy3.6"] + python-version: ["2.7", "3.4", "3.5", "3.6"] auth-ssl: noauth-nossl # TODO: dependency error for 'coverage-report' task: # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config @@ -2223,6 +2237,15 @@ buildvariants: display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" tasks: *encryption-server-versions +- matrix_name: "tests-pypy-debian-test-encryption" + matrix_spec: + platform: debian92 + python-version: ["pypy", "pypy3.5", "pypy3.6"] + auth-ssl: noauth-nossl + encryption: "*" + display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" + tasks: *encryption-server-versions + - matrix_name: "tests-python-version-rhel62-without-c-extensions" matrix_spec: platform: rhel62 @@ -2365,12 +2388,6 @@ buildvariants: python-version-windows: "*" auth-ssl: "*" encryption: "*" - exclude_spec: - # PYTHON-2366 Skip 3.9 due to cryptography install failures - - platform: "*" - python-version-windows: ["3.9"] - auth-ssl: "*" - encryption: "*" display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}" tasks: *encryption-server-versions @@ -2533,7 +2550,19 @@ buildvariants: - matrix_name: "ocsp-test" matrix_spec: platform: ubuntu-16.04 - python-version: ["2.7", "3.4", "3.8", "3.9", "pypy", "pypy3.5"] + python-version: ["2.7", "3.4", "3.8", "3.9"] + mongodb-version: ["4.4", "latest"] + auth: "noauth" + ssl: "ssl" + display_name: "OCSP test ${platform} ${python-version} ${mongodb-version}" + batchtime: 20160 # 14 days + tasks: + - name: ".ocsp" + +- matrix_name: "ocsp-test-pypy" + matrix_spec: + platform: debian92 + python-version: ["pypy", "pypy3.5", "pypy3.6"] mongodb-version: ["4.4", "latest"] auth: "noauth" ssl: "ssl" @@ -2545,7 +2574,7 @@ buildvariants: - matrix_name: "ocsp-test-windows" matrix_spec: platform: windows-64-vsMulti-small - python-version-windows: ["2.7", "3.4", "3.8"] + python-version-windows: ["2.7", "3.4", "3.9"] mongodb-version: ["4.4", "latest"] auth: "noauth" ssl: "ssl" diff --git a/.evergreen/run-ocsp-tests.sh b/.evergreen/run-ocsp-tests.sh index 17abb395c..75fba7e2c 100644 --- a/.evergreen/run-ocsp-tests.sh +++ b/.evergreen/run-ocsp-tests.sh @@ -3,6 +3,9 @@ set -o xtrace set -o errexit +# For createvirtualenv. +. .evergreen/utils.sh + if [ -z "$PYTHON_BINARY" ]; then echo "No python binary specified" PYTHON=$(command -v python || command -v python3) || true @@ -14,36 +17,9 @@ else PYTHON="$PYTHON_BINARY" fi -if $PYTHON -m virtualenv --version; then - VIRTUALENV="$PYTHON -m virtualenv" -elif command -v virtualenv; then - # We can remove this fallback after: - # https://github.com/10gen/mongo-python-toolchain/issues/8 - VIRTUALENV="$(command -v virtualenv) -p $PYTHON" -else - echo "Cannot test without virtualenv" - exit 1 -fi - -$VIRTUALENV --never-download --no-wheel ocsptest -if [ "Windows_NT" = "$OS" ]; then - . ocsptest/Scripts/activate -else - . ocsptest/bin/activate -fi +createvirtualenv $PYTHON ocsptest trap "deactivate; rm -rf ocsptest" EXIT HUP -IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')") -if [ $IS_PYTHON_2 = "1" ]; then - echo "Using a Python 2" - # Upgrade pip to install the cryptography wheel and not the tar. - # <20.1 because 20.0.2 says a future release may drop support for 2.7. - python -m pip install --upgrade 'pip<20.1' - # Upgrade setuptools because cryptography requires 18.5+. - # <45 because 45.0 dropped support for 2.7. - python -m pip install --upgrade 'setuptools<45' -fi - -python -m pip install pyopenssl requests service_identity +python -m pip install --prefer-binary pyopenssl requests service_identity OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED} CA_FILE=${CA_FILE} python test/ocsp/test_ocsp.py diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index c1a728643..66b126a60 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -94,38 +94,11 @@ fi # PyOpenSSL test setup. if [ -n "$TEST_PYOPENSSL" ]; then - if $PYTHON -m virtualenv --version; then - VIRTUALENV="$PYTHON -m virtualenv" - elif command -v virtualenv; then - # We can remove this fallback after: - # https://github.com/10gen/mongo-python-toolchain/issues/8 - VIRTUALENV="$(command -v virtualenv) -p $PYTHON" - else - echo "Cannot test without virtualenv" - exit 1 - fi - - $VIRTUALENV pyopenssltest - if [ "Windows_NT" = "$OS" ]; then - . pyopenssltest/Scripts/activate - else - . pyopenssltest/bin/activate - fi + createvirtualenv $PYTHON pyopenssltest trap "deactivate; rm -rf pyopenssltest" EXIT HUP PYTHON=python - IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')") - if [ $IS_PYTHON_2 = "1" ]; then - echo "Using a Python 2" - # Upgrade pip to install the cryptography wheel and not the tar. - # <20.1 because 20.0.2 says a future release may drop support for 2.7. - python -m pip install --upgrade 'pip<20.1' - # Upgrade setuptools because cryptography requires 18.5+. - # <45 because 45.0 dropped support for 2.7. - python -m pip install --upgrade 'setuptools<45' - fi - - python -m pip install pyopenssl requests service_identity + python -m pip install --prefer-binary pyopenssl requests service_identity fi if [ -n "$TEST_ENCRYPTION" ]; then @@ -166,7 +139,8 @@ if [ -n "$TEST_ENCRYPTION" ]; then # TODO: Test with 'pip install pymongocrypt' git clone --branch master https://github.com/mongodb/libmongocrypt.git libmongocrypt_git - python -m pip install --upgrade ./libmongocrypt_git/bindings/python + python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt + python -m pip install ./libmongocrypt_git/bindings/python python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" # PATH is updated by PREPARE_SHELL for access to mongocryptd. diff --git a/.evergreen/test-encryption-requirements.txt b/.evergreen/test-encryption-requirements.txt new file mode 100644 index 000000000..3b66938bb --- /dev/null +++ b/.evergreen/test-encryption-requirements.txt @@ -0,0 +1,4 @@ +# cffi==1.14.3 was the last installable release on RHEL 6.2 with Python 3.4 +cffi==1.14.3;python_version=="3.4" +cffi>=1.12.0,<2;python_version!="3.4" +cryptography>=2,<4 diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 9a92f1529..ff73be01d 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -8,19 +8,24 @@ createvirtualenv () { PYTHON=$1 VENVPATH=$2 if $PYTHON -m virtualenv --version; then - VIRTUALENV="$PYTHON -m virtualenv" + VIRTUALENV="$PYTHON -m virtualenv --never-download" + elif $PYTHON -m venv -h>/dev/null; then + VIRTUALENV="$PYTHON -m venv" elif command -v virtualenv; then - VIRTUALENV="$(command -v virtualenv) -p $PYTHON" + VIRTUALENV="$(command -v virtualenv) -p $PYTHON --never-download" else echo "Cannot test without virtualenv" exit 1 fi - $VIRTUALENV --system-site-packages --never-download $VENVPATH + $VIRTUALENV $VENVPATH if [ "Windows_NT" = "$OS" ]; then . $VENVPATH/Scripts/activate else . $VENVPATH/bin/activate fi + # Upgrade to the latest versions of pip setuptools wheel so that + # pip can always download the latest cryptography+cffi wheels. + python -m pip install --upgrade pip setuptools wheel } # Usage: