From 6f00a240c78ee3bbe97406f03b52da368fc6140d Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 12 Aug 2019 13:42:24 -0700 Subject: [PATCH] PYTHON-1955 Test encryption in evergreen Only enable xtrace output for non-sensitive tasks. --- .evergreen/config.yml | 262 +++++++++++---------- .evergreen/run-tests.sh | 65 ++++- pymongo/encryption.py | 5 +- pymongo/pool.py | 2 +- test/client-side-encryption/spec/bulk.json | 12 +- test/test_encryption.py | 29 +-- test/utils_spec_runner.py | 6 +- 7 files changed, 230 insertions(+), 151 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4346d3725..efd143bc9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -67,7 +67,6 @@ functions: PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" PREPARE_SHELL: | set -o errexit - set -o xtrace export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" @@ -90,6 +89,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS if [ "${project}" = "drivers-tools" ]; then @@ -100,95 +100,6 @@ functions: fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - "upload release": - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${project}.tar.gz - remote_file: ${UPLOAD_BUCKET}/${project}-${CURRENT_VERSION}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - - # Upload build artifacts that other tasks may depend on - # Note this URL needs to be totally unique, while predictable for the next task - # so it can automatically download the artifacts - "upload build": - # Compress and upload the entire build directory - - command: archive.targz_pack - params: - # Example: mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz - target: "${build_id}.tar.gz" - source_dir: ${PROJECT_DIRECTORY}/ - include: - - "./**" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${build_id}.tar.gz - # Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${task_name}/${build_id}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - - "fetch build": - - command: shell.exec - params: - continue_on_err: true - script: "set -o xtrace && rm -rf ${PROJECT_DIRECTORY}" - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz - bucket: mciuploads - local_file: build.tar.gz - - command: shell.exec - params: - continue_on_err: true - # EVG-1105: Use s3.get extract_to: ./ - script: "set -o xtrace && cd .. && rm -rf ${PROJECT_DIRECTORY} && mkdir ${PROJECT_DIRECTORY}/ && tar xf build.tar.gz -C ${PROJECT_DIRECTORY}/" - - "exec compile script" : - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - [ -f ${PROJECT_DIRECTORY}/${file} ] && BUILDTOOL="${buildtool}" sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping" - - "exec script" : - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - [ -f ${PROJECT_DIRECTORY}/${file} ] && sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping" - - "upload docs" : - - command: shell.exec - params: - silent: true - script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://mciuploads/${UPLOAD_BUCKET}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1 - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/doc/html/index.html - remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: "Rendered docs" - "upload coverage" : - command: s3.put params: @@ -218,6 +129,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} # Coverage combine merges (and removes) all the coverage files and # generates a new .coverage file in the current directory. @@ -245,39 +157,12 @@ functions: content_type: text/html display_name: "Coverage Report HTML" - "upload scan artifacts" : - - command: shell.exec - type: test - params: - script: | - cd - if find ${PROJECT_DIRECTORY}/scan -name \*.html | grep -q html; then - (cd ${PROJECT_DIRECTORY}/scan && find . -name index.html -exec echo "
  • {}
    • " \;) >> scan.html - else - echo "No issues found" > scan.html - fi - - command: shell.exec - params: - silent: true - script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/scan s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1 - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/scan.html - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: "Scan Build Report" "upload mo artifacts": - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - command: s3.put @@ -350,6 +235,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} MONGODB_VERSION=${VERSION} \ TOPOLOGY=${TOPOLOGY} \ @@ -372,6 +258,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh @@ -381,6 +268,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh @@ -390,6 +278,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mockupdb-tests.sh @@ -399,6 +288,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} sh ${PROJECT_DIRECTORY}/.evergreen/run-cdecimal-tests.sh @@ -408,6 +298,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} sh ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh @@ -415,8 +306,24 @@ functions: - command: shell.exec type: test params: + silent: true working_dir: "src" script: | + if [ -n "${test_encryption}" ]; then + cat < fle_aws_creds.sh + export FLE_AWS_KEY="${fle_aws_key}" + export FLE_AWS_SECRET="${fle_aws_secret}" + EOT + fi + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + if [ -n "${set_xtrace_on}" ]; then + set -o xtrace + export SET_XTRACE_ON="${set_xtrace_on}" + fi ${PREPARE_SHELL} if [ -n "${MONGODB_STARTED}" ]; then export PYMONGO_MUST_CONNECT=1 @@ -424,6 +331,14 @@ functions: if [ -n "${DISABLE_TEST_COMMANDS}" ]; then export PYMONGO_DISABLE_TEST_COMMANDS=1 fi + if [ -n "${test_encryption}" ]; then + # Disable xtrace (just in case it was accidentally set). + set +x + . ./fle_aws_creds.sh + rm -f ./fle_aws_creds.sh + export LIBMONGOCRYPT_URL="${libmongocrypt_url}" + export TEST_ENCRYPTION=1 + fi PYTHON_BINARY=${PYTHON_BINARY} \ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ @@ -458,6 +373,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS || true @@ -465,6 +381,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename @@ -474,6 +391,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do cat $i | tr -d '\r' > $i.new @@ -486,6 +404,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do chmod +x $i @@ -495,6 +414,7 @@ functions: - command: shell.exec params: script: | + set -o xtrace ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json @@ -503,6 +423,7 @@ functions: params: working_dir: "src" script: | + set -o xtrace ${PREPARE_SHELL} file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. @@ -871,6 +792,8 @@ axes: display_name: "Amazon Linux 2018 (Enterprise)" run_on: amazon1-2018-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/linux-64-amazon-ami/master/latest/libmongocrypt.tar.gz - id: archlinux-test display_name: "Archlinux" run_on: archlinux-test @@ -887,17 +810,25 @@ axes: display_name: "Debian 9.2" run_on: debian92-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian92/master/latest/libmongocrypt.tar.gz - id: macos-1012 display_name: "macOS 10.12" run_on: macos-1012 + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: rhel62 display_name: "RHEL 6.2 (x86_64)" run_on: rhel62-small batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-62-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel70 display_name: "RHEL 7.0" run_on: rhel70-small batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel71-power8-test display_name: "RHEL 7.1 (POWER8)" run_on: rhel71-power8-test @@ -906,10 +837,14 @@ axes: display_name: "RHEL 7.2 (zSeries)" run_on: rhel72-zseries-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel72-zseries-test/master/latest/libmongocrypt.tar.gz - id: suse12-x86-64-test display_name: "SUSE 12 (x86_64)" run_on: suse12-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/suse12-64/master/latest/libmongocrypt.tar.gz - id: ubuntu-12.04 display_name: "Ubuntu 12.04" run_on: ubuntu1204-test @@ -918,10 +853,14 @@ axes: display_name: "Ubuntu 16.04" run_on: ubuntu1604-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1604/master/latest/libmongocrypt.tar.gz - id: ubuntu1604-arm64-small display_name: "Ubuntu 16.04 (ARM64)" run_on: ubuntu1604-arm64-small batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1604-arm64/master/latest/libmongocrypt.tar.gz - id: ubuntu1604-power8-test display_name: "Ubuntu 16.04 (POWER8)" run_on: ubuntu1604-power8-test @@ -930,14 +869,20 @@ axes: display_name: "Ubuntu 18.04 (ARM64)" run_on: ubuntu1804-arm64-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1804-arm64/master/latest/libmongocrypt.tar.gz - id: windows-vs2010 display_name: "Windows 64 Visual Studio 2010" run_on: windows-64-vs2010-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz - id: windows-vs2015 display_name: "Windows 64 Visual Studio 2015" run_on: windows-64-vs2015-test batchtime: 10080 # 7 days + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz # Test with authentication? - id: auth @@ -1134,6 +1079,15 @@ axes: variables: COVERAGE: "coverage" + # Run encryption tests? + - id: encryption + display_name: "Encryption" + values: + - id: "encryption" + display_name: "Encryption" + tags: ["encryption_tag"] + variables: + test_encryption: true buildvariants: - matrix_name: "tests-all" @@ -1154,6 +1108,21 @@ buildvariants: - ".3.0" - ".2.6" +- matrix_name: "tests-all-encryption" + matrix_spec: + platform: + # OSes that support versions of MongoDB>=2.6 with SSL. + - awslinux + - rhel70 + auth-ssl: "*" + encryption: "*" + display_name: "Encryption ${platform} ${auth-ssl}" + tasks: &encryption-server-versions + - ".4.2" + # TODO: PYTHON-1926 + # - ".4.0" + # - ".2.6" + - matrix_name: "tests-no-36-plus" matrix_spec: platform: @@ -1252,6 +1221,16 @@ buildvariants: - ".3.0" - ".2.6" +- matrix_name: "test-macos-encryption" + matrix_spec: + platform: + - macos-1012 + auth: "auth" + ssl: "nossl" + encryption: "*" + display_name: "Encryption ${platform} ${auth} ${ssl}" + tasks: *encryption-server-versions + - matrix_name: "test-os-requires-34-no-42plus" matrix_spec: platform: @@ -1290,6 +1269,8 @@ buildvariants: tasks: - ".latest" - ".4.2" + variables: + set_xtrace_on: on - matrix_name: "tests-python-version-rhel62-test-ssl" matrix_spec: @@ -1320,6 +1301,19 @@ buildvariants: display_name: "${python-version} ${platform} ${auth} ${ssl} ${coverage}" tasks: *all-server-versions +- matrix_name: "tests-python-version-rhel62-test-encryption" + matrix_spec: + platform: rhel62 + # RHEL 6.2 does not support Python 3.7.x and later. + python-version: ["2.7", "3.4", "3.5", "3.6", "pypy", "pypy3.5"] + auth-ssl: noauth-nossl +# TODO: dependency error for 'coverage-report' task: +# dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config +# coverage: "*" + encryption: "*" + display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" + tasks: *encryption-server-versions + - matrix_name: "tests-python-version-rhel62-without-c-extensions" matrix_spec: platform: rhel62 @@ -1410,6 +1404,16 @@ buildvariants: display_name: "${platform} ${python-version} ${auth-ssl}" tasks: *all-server-versions +# windows-vs2010 3.4 is unable to dlopen the libmongocrypt ddl built on 2016 +#- matrix_name: "tests-windows-vs2010-python-version-encryption" +# matrix_spec: +# platform: windows-vs2010 +# python-version: *win-vs2010-pythons +# auth-ssl: "*" +# encryption: "*" +# display_name: "Encryption ${platform} ${python-version} ${auth-ssl}" +# tasks: *encryption-server-versions + - matrix_name: "tests-python-version-requires-openssl-102-plus-test-ssl" matrix_spec: platform: ubuntu-16.04 @@ -1424,6 +1428,15 @@ buildvariants: - ".3.4" - ".3.2" +- matrix_name: "tests-python-version-requires-openssl-102-plus-test-encryption" + matrix_spec: + platform: ubuntu-16.04 + python-version: *openssl-102-plus-pythons + auth-ssl: "noauth-nossl" + encryption: "*" + display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" + tasks: *encryption-server-versions + - matrix_name: "tests-python-version-supports-openssl-110-test-ssl" matrix_spec: platform: debian92 @@ -1443,6 +1456,15 @@ buildvariants: display_name: "${platform} ${python-version} ${auth-ssl}" tasks: *all-server-versions +- matrix_name: "tests-windows-vs2015-python-version-encryption" + matrix_spec: + platform: windows-vs2015 + python-version: *win-vs2015-pythons + auth-ssl: "*" + encryption: "*" + display_name: "Encryption ${platform} ${python-version} ${auth-ssl}" + tasks: *encryption-server-versions + # Storage engine tests on RHEL 6.2 (x86_64) with Python 2.7. - matrix_name: "tests-storage-engines" matrix_spec: @@ -1562,6 +1584,8 @@ buildvariants: - rhel62-small tasks: - name: "no-server" + expansions: + set_xtrace_on: on - name: "Coverage Report" display_name: "Coverage Report" @@ -1569,6 +1593,8 @@ buildvariants: - ubuntu1604-test tasks: - name: "coverage-report" + expansions: + set_xtrace_on: on - matrix_name: "atlas-connect" matrix_spec: diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6248bfe84..460540a8e 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,14 +1,22 @@ #!/bin/sh -set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail # Supported/used environment variables: -# AUTH Set to enable authentication. Defaults to "noauth" -# SSL Set to enable SSL. Defaults to "nossl" -# PYTHON_BINARY The Python version to use. Defaults to whatever is available -# GREEN_FRAMEWORK The green framework to test with, if any. -# C_EXTENSIONS Pass --no_ext to setup.py, or not. -# COVERAGE If non-empty, run the test suite with coverage. +# SET_XTRACE_ON Set to non-empty to write all commands first to stderr. +# AUTH Set to enable authentication. Defaults to "noauth" +# SSL Set to enable SSL. Defaults to "nossl" +# PYTHON_BINARY The Python version to use. Defaults to whatever is available +# GREEN_FRAMEWORK The green framework to test with, if any. +# C_EXTENSIONS Pass --no_ext to setup.py, or not. +# COVERAGE If non-empty, run the test suite with coverage. +# TEST_ENCRYPTION If non-empty, install pymongocrypt. +# LIBMONGOCRYPT_URL The URL to download libmongocrypt. + +if [ -n "${SET_XTRACE_ON}" ]; then + set -o xtrace +else + set +x +fi AUTH=${AUTH:-noauth} @@ -18,8 +26,10 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-} C_EXTENSIONS=${C_EXTENSIONS:-} COVERAGE=${COVERAGE:-} COMPRESSORS=${COMPRESSORS:-} +TEST_ENCRYPTION=${TEST_ENCRYPTION:-} +LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-} -if [ -n $COMPRESSORS ]; then +if [ -n "$COMPRESSORS" ]; then export COMPRESSORS=$COMPRESSORS fi @@ -49,14 +59,14 @@ if [ -z "$PYTHON_BINARY" ]; then PYTHON=python trap "deactivate; rm -rf pymongotestvenv" EXIT HUP fi -elif [ $COMPRESSORS = "snappy" ]; then +elif [ "$COMPRESSORS" = "snappy" ]; then $PYTHON_BINARY -m virtualenv --system-site-packages --never-download snappytest . snappytest/bin/activate trap "deactivate; rm -rf snappytest" EXIT HUP # 0.5.2 has issues in pypy3(.5) pip install python-snappy==0.5.1 PYTHON=python -elif [ $COMPRESSORS = "zstd" ]; then +elif [ "$COMPRESSORS" = "zstd" ]; then $PYTHON_BINARY -m virtualenv --system-site-packages --never-download zstdtest . zstdtest/bin/activate trap "deactivate; rm -rf zstdtest" EXIT HUP @@ -66,6 +76,41 @@ else PYTHON="$PYTHON_BINARY" fi +if [ -n "$TEST_ENCRYPTION" ]; then + if [ -z "$LIBMONGOCRYPT_URL" ]; then + echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!" + exit 1 + fi + curl -O "$LIBMONGOCRYPT_URL" + mkdir libmongocrypt + tar xzf libmongocrypt.tar.gz -C ./libmongocrypt + ls -la libmongocrypt + ls -la libmongocrypt/nocrypto + # Use the nocrypto build to avoid dependency issues with older windows/python versions. + BASE=$(pwd)/libmongocrypt/nocrypto + if [ -f "${BASE}/lib/libmongocrypt.so" ]; then + export PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so + elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then + export PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib + elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then + PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll + # libmongocrypt's windows dll is not marked executable. + chmod +x $PYMONGOCRYPT_LIB + export PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) + elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then + export PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so + else + echo "Cannot find libmongocrypt shared object file" + exit 1 + fi + + git clone --branch master git@github.com:mongodb/libmongocrypt.git libmongocrypt_git + $PYTHON -m pip install --upgrade ./libmongocrypt_git/bindings/python + $PYTHON -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" + $PYTHON -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" + # PATH is set by PREPARE_SHELL. +fi + PYTHON_IMPL=$($PYTHON -c "import platform, sys; sys.stdout.write(platform.python_implementation())") if [ $PYTHON_IMPL = "Jython" ]; then EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m" diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 8d30edc37..db874453e 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -117,11 +117,14 @@ class _EncryptionIO(MongoCryptCallback): opts = PoolOptions(connect_timeout=_KMS_CONNECT_TIMEOUT, socket_timeout=_KMS_CONNECT_TIMEOUT, ssl_context=ctx) - with _configured_socket((endpoint, _HTTPS_PORT), opts) as conn: + conn = _configured_socket((endpoint, _HTTPS_PORT), opts) + try: conn.sendall(message) while kms_context.bytes_needed > 0: data = conn.recv(kms_context.bytes_needed) kms_context.feed(data) + finally: + conn.close() def collection_info(self, database, filter): """Get the collection info for a namespace. diff --git a/pymongo/pool.py b/pymongo/pool.py index 5cb7ddf52..1ccaf700e 100644 --- a/pymongo/pool.py +++ b/pymongo/pool.py @@ -794,7 +794,7 @@ class SocketInfo(object): if isinstance(error, socket.error): _raise_connection_failure(self.address, error) else: - raise error + raise def __eq__(self, other): return self.sock == other.sock diff --git a/test/client-side-encryption/spec/bulk.json b/test/client-side-encryption/spec/bulk.json index 5e71b593b..232635609 100644 --- a/test/client-side-encryption/spec/bulk.json +++ b/test/client-side-encryption/spec/bulk.json @@ -148,8 +148,16 @@ "name": "deleteOne", "arguments": { "filter": { - "encrypted_string": "string1", - "_id": 2 + "$and": [ + { + "encrypted_string": "string1" + }, + { + "_id": { + "$eq": 2 + } + } + ] } } } diff --git a/test/test_encryption.py b/test/test_encryption.py index f718c7436..54192d48a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -31,7 +31,6 @@ from bson.binary import (Binary, from bson.codec_options import CodecOptions from bson.errors import BSONError from bson.json_util import JSONOptions -from bson.raw_bson import RawBSONDocument from bson.son import SON from pymongo.errors import (ConfigurationError, @@ -172,11 +171,9 @@ class TestClientSimple(EncryptionIntegrationTest): self.addCleanup(client.close) # Create the encrypted field's data key. - key_vault = self.client.admin.get_collection( - 'datakeys', codec_options=OPTS) - data_key = RawBSONDocument( - bson_data('custom', 'key-document-local.json')) - key_vault.insert_one(data_key) + key_vault = create_key_vault( + self.client.admin.datakeys, + json_data('custom', 'key-document-local.json')) self.addCleanup(key_vault.drop) # Collection.insert_one/insert_many auto encrypts. @@ -219,7 +216,7 @@ class TestClientSimple(EncryptionIntegrationTest): # Collection.distinct auto decrypts. decrypted_ssns = encrypted_coll.distinct('ssn') - self.assertEqual(decrypted_ssns, [d['ssn'] for d in docs]) + self.assertEqual(set(decrypted_ssns), set(d['ssn'] for d in docs)) # Make sure the field is actually encrypted. for encrypted_doc in self.db.test.find(): @@ -233,9 +230,8 @@ class TestClientSimple(EncryptionIntegrationTest): def test_auto_encrypt(self): # Configure the encrypted field via jsonSchema. json_schema = json_data('custom', 'schema.json') - coll = self.db.create_collection( - 'test', validator={'$jsonSchema': json_schema}, codec_options=OPTS) - self.addCleanup(coll.drop) + create_with_schema(self.db.test, json_schema) + self.addCleanup(self.db.test.drop) opts = AutoEncryptionOpts(KMS_PROVIDERS, 'admin.datakeys') self._test_auto_encrypt(opts) @@ -427,9 +423,14 @@ class TestSpec(SpecRunner): def maybe_skip_scenario(self, test): super(TestSpec, self).maybe_skip_scenario(test) - if 'type=symbol' in test['description'].lower(): - raise unittest.SkipTest( - 'PyMongo does not support the symbol type') + desc = test['description'].lower() + if 'type=symbol' in desc: + self.skipTest('PyMongo does not support the symbol type') + if desc == 'explain a find with deterministic encryption': + # PyPy and Python 3.6+ have ordered dict. + if sys.version_info[:2] < (3, 6) and 'PyPy' not in sys.version: + self.skipTest( + 'explain test does not work without ordered dict') def setup_scenario(self, scenario_def): """Override a test's setup.""" @@ -496,7 +497,7 @@ AWS_KEY_ID = Binary( def create_with_schema(coll, json_schema): """Create and return a Collection with a jsonSchema.""" - coll.drop() + coll.with_options(write_concern=WriteConcern(w='majority')).drop() return coll.database.create_collection( coll.name, validator={'$jsonSchema': json_schema}, codec_options=OPTS) diff --git a/test/utils_spec_runner.py b/test/utils_spec_runner.py index c9b722437..c258d3548 100644 --- a/test/utils_spec_runner.py +++ b/test/utils_spec_runner.py @@ -607,11 +607,7 @@ def end_sessions(sessions): s.end_session() -if sys.version_info[:2] >= (3, 6): - DOC_CLASS = dict -else: - DOC_CLASS = SON -OPTS = CodecOptions(document_class=DOC_CLASS, uuid_representation=STANDARD) +OPTS = CodecOptions(document_class=dict, uuid_representation=STANDARD) def decode_raw(val):