mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-4721 - Create individualized scripts for all shell.exec commands (#1997)

Browse Source 
Co-authored-by: Jib <jib.adegunloye@mongodb.com>
This commit is contained in:
Noah Stapp 2024-11-20 14:47:28 -05:00 committed by GitHub
parent 906d021bb1
commit 1c7a7fe9ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
40 changed files with 681 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

706
.evergreen/config.yml
View File

@ -20,10 +20,9 @@ exec_timeout_secs: 3600 # 60 minutes is the longest we'll ever run (primarily
# What to do when evergreen hits the timeout (`post:` tasks are run automatically)
timeout:
- command: shell.exec
- command: subprocess.exec
params:
script: |
ls -la
binary: ls -la
include:
- filename: .evergreen/generated_configs/tasks.yml
@ -41,7 +40,7 @@ functions:
# Make an evergreen expansion file with dynamic values
- command: subprocess.exec
params:
include_expansions_in_env: ["is_patch", "project", "version_id"]
include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"]
binary: bash
working_dir: "src"
args:
@ -52,19 +51,11 @@ functions:
file: src/expansion.yml
"prepare resources":
- command: shell.exec
- command: subprocess.exec
params:
script: |
. src/.evergreen/scripts/env.sh
set -o xtrace
rm -rf $DRIVERS_TOOLS
if [ "$PROJECT" = "drivers-tools" ]; then
# If this was a patch build, doing a fresh clone would not actually test the patch
cp -R ${PROJECT_DIRECTORY}/ ${DRIVERS_TOOLS}
else
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
fi
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
binary: bash
args:
- src/.evergreen/scripts/prepare-resources.sh
"upload coverage" :
- command: ec2.assume_role
@ -88,14 +79,17 @@ functions:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
- command: subprocess.exec
params:
silent: true
binary: bash
working_dir: "src"
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
script: |
# Download all the task coverage files.
aws s3 cp --recursive s3://${bucket_name}/coverage/${revision}/${version_id}/coverage/ coverage/
args:
- .evergreen/scripts/download-and-merge-coverage.sh
- ${bucket_name}
- ${revision}
- ${version_id}
- command: subprocess.exec
params:
working_dir: "src"
@ -103,13 +97,17 @@ functions:
args:
- .evergreen/combine-coverage.sh
# Upload the resulting html coverage report.
- command: shell.exec
- command: subprocess.exec
params:
silent: true
binary: bash
working_dir: "src"
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
script: |
aws s3 cp htmlcov/ s3://${bucket_name}/coverage/${revision}/${version_id}/htmlcov/ --recursive --acl public-read --region us-east-1
args:
- .evergreen/scripts/upload-coverage-report.sh
- ${bucket_name}
- ${revision}
- ${version_id}
# Attach the index.html with s3.put so it shows up in the Evergreen UI.
- command: s3.put
params:
@ -128,15 +126,6 @@ functions:
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
params:
script: |
. src/.evergreen/scripts/env.sh
set -o xtrace
mkdir out_dir
find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
tar zcvf mongodb-logs.tar.gz -C out_dir/ .
rm -rf out_dir
- command: archive.targz_pack
params:
target: "mongo-coredumps.tgz"
@ -161,23 +150,12 @@ functions:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: mongodb-logs.tar.gz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz
local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "mongodb-logs.tar.gz"
- command: s3.put
params:
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: drivers-tools/.evergreen/orchestration/server.log
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
bucket: ${bucket_name}
permissions: public-read
content_type: ${content_type|text/plain}
display_name: "orchestration.log"
display_name: "drivers-tools-logs.tar.gz"
"upload working dir":
- command: ec2.assume_role
@ -230,54 +208,13 @@ functions:
file: "src/xunit-results/TEST-*.xml"
"bootstrap mongo-orchestration":
- command: shell.exec
- command: subprocess.exec
params:
script: |
. src/.evergreen/scripts/env.sh
set -o xtrace
# Enable core dumps if enabled on the machine
# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
if [ -f /proc/self/coredump_filter ]; then
# Set the shell process (and its children processes) to dump ELF headers (bit 4),
# anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
echo 0x13 > /proc/self/coredump_filter
if [ -f /sbin/sysctl ]; then
# Check that the core pattern is set explicitly on our distro image instead
# of being the OS's default value. This ensures that coredump names are consistent
# across distros and can be picked up by Evergreen.
core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
if [ "$core_pattern" = "dump_%e.%p.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
fi
if [ $(uname -s) = "Darwin" ]; then
core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
if [ -n "${skip_crypt_shared}" ]; then
export SKIP_CRYPT_SHARED=1
fi
MONGODB_VERSION=${VERSION} \
TOPOLOGY=${TOPOLOGY} \
AUTH=${AUTH} \
SSL=${SSL} \
STORAGE_ENGINE=${STORAGE_ENGINE} \
DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \
ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
LOAD_BALANCER=${LOAD_BALANCER} \
bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
# run-orchestration generates expansion file with the MONGODB_URI for the cluster
binary: bash
include_expansions_in_env: ["VERSION", "TOPOLOGY", "AUTH", "SSL", "ORCHESTRATION_FILE", "LOAD_BALANCER"]
args:
- src/.evergreen/scripts/run-with-env.sh
- src/.evergreen/scripts/bootstrap-mongo-orchestration.sh
- command: expansions.update
params:
file: mo-expansion.yml
@ -288,167 +225,107 @@ functions:
value: "1"
"bootstrap data lake":
- command: shell.exec
- command: subprocess.exec
type: setup
params:
script: |
. src/.evergreen/scripts/env.sh
bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
- command: shell.exec
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
- command: subprocess.exec
type: setup
params:
script: |
. src/.evergreen/scripts/env.sh
bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
sleep 1
docker ps
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
"stop mongo-orchestration":
- command: shell.exec
- command: subprocess.exec
params:
script: |
. src/.evergreen/scripts/env.sh
set -o xtrace
bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
"run mod_wsgi tests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
include_expansions_in_env: [MOD_WSGI_VERSION, MOD_WSGI_EMBEDDED, "PYTHON_BINARY"]
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
set -o xtrace
PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} \
MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh
binary: bash
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mod-wsgi-tests.sh
"run mockupdb tests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["PYTHON_BINARY"]
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
set -o xtrace
export PYTHON_BINARY=${PYTHON_BINARY}
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb
binary: bash
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mockupdb-tests.sh
"run doctests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
include_expansions_in_env: [ "PYTHON_BINARY" ]
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
set -o xtrace
PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test
binary: bash
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-doctests.sh
"run tests":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: ["TEST_DATA_LAKE", "AUTH", "SSL", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
binary: bash
working_dir: "src"
shell: bash
background: true
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
script: |
. .evergreen/scripts/env.sh
if [ -n "${test_encryption}" ]; then
./.evergreen/hatch.sh encryption:setup
fi
- command: shell.exec
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
params:
working_dir: "src"
binary: bash
background: true
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/setup-encryption.sh
- command: subprocess.exec
type: test
params:
working_dir: "src"
shell: bash
script: |
# Disable xtrace
set +x
. .evergreen/scripts/env.sh
if [ -n "${MONGODB_STARTED}" ]; then
export PYMONGO_MUST_CONNECT=true
fi
if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
export PYMONGO_DISABLE_TEST_COMMANDS=1
fi
if [ -n "${test_encryption}" ]; then
# Disable xtrace (just in case it was accidentally set).
set +x
bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
export TEST_ENCRYPTION=1
if [ -n "${test_encryption_pyopenssl}" ]; then
export TEST_ENCRYPTION_PYOPENSSL=1
fi
fi
if [ -n "${test_crypt_shared}" ]; then
export TEST_CRYPT_SHARED=1
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
fi
if [ -n "${test_pyopenssl}" ]; then
export TEST_PYOPENSSL=1
fi
if [ -n "${SETDEFAULTENCODING}" ]; then
export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
fi
if [ -n "${test_loadbalancer}" ]; then
export TEST_LOADBALANCER=1
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
fi
if [ -n "${test_serverless}" ]; then
export TEST_SERVERLESS=1
fi
if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then
export TEST_INDEX_MANAGEMENT=1
fi
if [ -n "${SKIP_CSOT_TESTS}" ]; then
export SKIP_CSOT_TESTS=1
fi
GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
PYTHON_BINARY=${PYTHON_BINARY} \
NO_EXT=${NO_EXT} \
COVERAGE=${COVERAGE} \
COMPRESSORS=${COMPRESSORS} \
AUTH=${AUTH} \
SSL=${SSL} \
TEST_DATA_LAKE=${TEST_DATA_LAKE} \
TEST_SUITES=${TEST_SUITES} \
MONGODB_API_VERSION=${MONGODB_API_VERSION} \
SKIP_HATCH=${SKIP_HATCH} \
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
binary: bash
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"]
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-tests.sh
"run enterprise auth tests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: "src"
include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
script: |
# Disable xtrace for security reasons (just in case it was accidentally set).
set +x
bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth
PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \
PYTHON_BINARY="${PYTHON_BINARY}" \
TEST_ENTERPRISE_AUTH=1 \
AUTH=auth \
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY"]
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-enterprise-auth-tests.sh
"run atlas tests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
binary: bash
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY"]
working_dir: "src"
script: |
# Disable xtrace for security reasons (just in case it was accidentally set).
set +x
set -o errexit
bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \
PYTHON_BINARY="${PYTHON_BINARY}" \
TEST_ATLAS=1 \
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-atlas-tests.sh
"get aws auth secrets":
- command: subprocess.exec
@ -460,57 +337,140 @@ functions:
- ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh
"run aws auth test with regular aws credentials":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
shell: "bash"
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh regular
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- regular
"run aws auth test with assume role credentials":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
shell: "bash"
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh assume-role
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- assume-role
"run aws auth test with aws EC2 credentials":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
shell: "bash"
script: |
if [ "${skip_EC2_auth_test}" = "true" ]; then
echo "This platform does not support the EC2 auth test, skipping..."
exit 0
fi
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh ec2
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- ec2
"run aws auth test with aws web identity credentials":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- # Test with and without AWS_ROLE_SESSION_NAME set.
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
shell: "bash"
script: |
if [ "${skip_EC2_auth_test}" = "true" ]; then
echo "This platform does not support the web identity auth test, skipping..."
exit 0
fi
. .evergreen/scripts/env.sh
# Test with and without AWS_ROLE_SESSION_NAME set.
.evergreen/run-mongodb-aws-test.sh web-identity
AWS_ROLE_SESSION_NAME="test" \
.evergreen/run-mongodb-aws-test.sh web-identity
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- web-identity
- command: subprocess.exec
type: test
params:
include_expansions_in_env: [ "DRIVERS_TOOLS", "skip_EC2_auth_test" ]
binary: bash
working_dir: "src"
env:
AWS_ROLE_SESSION_NAME: test
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- web-identity
"run aws auth test with aws credentials as environment variables":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- env-creds
"run aws auth test with aws credentials and session token as environment variables":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-mongodb-aws-test.sh
- session-creds
"run oidc auth test with test credentials":
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
@ -532,110 +492,69 @@ functions:
args:
- ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh
"run aws auth test with aws credentials as environment variables":
- command: shell.exec
type: test
params:
working_dir: "src"
shell: bash
script: |
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh env-creds
"run aws auth test with aws credentials and session token as environment variables":
- command: shell.exec
type: test
params:
working_dir: "src"
shell: bash
script: |
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh session-creds
"run aws ECS auth test":
- command: shell.exec
- command: subprocess.exec
type: test
params:
shell: "bash"
binary: bash
working_dir: "src"
script: |
if [ "${skip_ECS_auth_test}" = "true" ]; then
echo "This platform does not support the ECS auth test, skipping..."
exit 0
fi
. .evergreen/scripts/env.sh
set -ex
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
. ./activate-authawsvenv.sh
. aws_setup.sh ecs
export MONGODB_BINARIES="$MONGODB_BINARIES";
export PROJECT_DIRECTORY="${PROJECT_DIRECTORY}";
python aws_tester.py ecs
cd -
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-aws-ecs-auth-test.sh
"cleanup":
- command: shell.exec
- command: subprocess.exec
params:
binary: bash
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
if [ -f $DRIVERS_TOOLS/.evergreen/csfle/secrets-export.sh ]; then
. .evergreen/hatch.sh encryption:teardown
fi
rm -rf ${DRIVERS_TOOLS} || true
rm -f ./secrets-export.sh || true
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/cleanup.sh
"teardown":
- command: subprocess.exec
params:
binary: bash
working_dir: "src"
args:
- ${DRIVERS_TOOLS}/.evergreen/teardown.sh
"fix absolute paths":
- command: shell.exec
- command: subprocess.exec
params:
script: |
set +x
. src/.evergreen/scripts/env.sh
for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename
done
binary: bash
args:
- src/.evergreen/scripts/fix-absolute-paths.sh
"windows fix":
- command: shell.exec
- command: subprocess.exec
params:
script: |
set +x
. src/.evergreen/scripts/env.sh
for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
cat $i | tr -d '\r' > $i.new
mv $i.new $i
done
# Copy client certificate because symlinks do not work on Windows.
cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem
binary: bash
args:
- src/.evergreen/scripts/windows-fix.sh
"make files executable":
- command: shell.exec
- command: subprocess.exec
params:
script: |
set +x
. src/.evergreen/scripts/env.sh
for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
chmod +x $i
done
binary: bash
args:
- src/.evergreen/scripts/make-files-executable.sh
"init test-results":
- command: shell.exec
- command: subprocess.exec
params:
script: |
set +x
. src/.evergreen/scripts/env.sh
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json
binary: bash
args:
- src/.evergreen/scripts/init-test-results.sh
"install dependencies":
- command: shell.exec
- command: subprocess.exec
params:
binary: bash
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
set -o xtrace
file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh"
# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
[ -f "$file" ] && bash $file || echo "$file not available, skipping"
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/install-dependencies.sh
"assume ec2 role":
- command: ec2.assume_role
@ -657,18 +576,22 @@ functions:
file: atlas-expansion.yml
"run-ocsp-test":
- command: shell.exec
- command: subprocess.exec
params:
include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ]
binary: bash
working_dir: "src"
args:
- .evergreen/scripts/setup-tests.sh
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED", "PYTHON_BINARY"]
binary: bash
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
TEST_OCSP=1 \
PYTHON_BINARY=${PYTHON_BINARY} \
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
bash ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-ocsp-test.sh
"run-ocsp-server":
- command: subprocess.exec
@ -680,42 +603,38 @@ functions:
- ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh
"run load-balancer":
- command: shell.exec
- command: subprocess.exec
params:
script: |
DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
binary: bash
include_expansions_in_env: ["MONGODB_URI"]
args:
- src/.evergreen/scripts/run-with-env.sh
- src/.evergreen/scripts/run-load-balancer.sh
- command: expansions.update
params:
file: lb-expansion.yml
"stop load-balancer":
- command: shell.exec
- command: subprocess.exec
params:
script: |
cd ${DRIVERS_TOOLS}/.evergreen
DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
binary: bash
args:
- src/.evergreen/scripts/stop-load-balancer.sh
"teardown_docker":
- command: shell.exec
- command: subprocess.exec
params:
script: |
# Remove all Docker images
DOCKER=$(command -v docker) || true
if [ -n "$DOCKER" ]; then
docker rmi -f $(docker images -a -q) &> /dev/null || true
fi
binary: bash
args:
- src/.evergreen/scripts/teardown-docker.sh
"teardown_aws":
- command: shell.exec
- command: subprocess.exec
params:
shell: "bash"
script: |
. src/.evergreen/scripts/env.sh
cd "${DRIVERS_TOOLS}/.evergreen/auth_aws"
if [ -f "./aws_e2e_setup.json" ]; then
. ./activate-authawsvenv.sh
python ./lib/aws_assign_instance_profile.py
fi
binary: bash
args:
- src/.evergreen/scripts/run-with-env.sh
- src/.evergreen/scripts/teardown-aws.sh
"teardown atlas":
- command: subprocess.exec
@ -725,13 +644,14 @@ functions:
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
"run perf tests":
- command: shell.exec
- command: subprocess.exec
type: test
params:
working_dir: "src"
script: |
. .evergreen/scripts/env.sh
PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh
binary: bash
args:
- .evergreen/scripts/run-with-env.sh
- .evergreen/scripts/run-perf-tests.sh
"attach benchmark test results":
- command: attach.results
@ -756,6 +676,7 @@ pre:
post:
# Disabled, causing timeouts
# - func: "upload working dir"
- func: "teardown"
- func: "upload coverage"
- func: "upload mo artifacts"
- func: "upload test results"
@ -798,13 +719,13 @@ task_groups:
- func: make files executable
- command: subprocess.exec
params:
binary: "bash"
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
teardown_task:
- command: subprocess.exec
params:
binary: "bash"
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh
- func: "upload test results"
@ -966,31 +887,12 @@ tasks:
# Throw it here, and execute this task on all buildvariants
- name: getdata
commands:
- command: shell.exec
- command: subprocess.exec
binary: bash
type: test
params:
script: |
set -o xtrace
. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true
get_distro || true
echo $DISTRO
echo $MARCH
echo $OS
uname -a || true
ls /etc/*release* || true
cc --version || true
gcc --version || true
clang --version || true
gcov --version || true
lcov --version || true
llvm-cov --version || true
echo $PATH
ls -la /usr/local/Cellar/llvm/*/bin/ || true
ls -la /usr/local/Cellar/ || true
scan-build --version || true
genhtml --version || true
valgrind --version || true
args:
- src/.evergreen/scripts/run-getdata.sh
# Standard test tasks {{{
- name: "mockupdb"
@ -1647,7 +1549,7 @@ tasks:
type: setup
params:
working_dir: "src"
binary: "bash"
binary: bash
include_expansions_in_env: ["DRIVERS_TOOLS"]
args:
- .evergreen/run-gcpkms-test.sh
@ -1660,17 +1562,14 @@ tasks:
vars:
VERSION: "latest"
TOPOLOGY: "server"
- command: shell.exec
- command: subprocess.exec
type: test
params:
include_expansions_in_env: ["PYTHON_BINARY"]
working_dir: "src"
shell: "bash"
script: |
. .evergreen/scripts/env.sh
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg
binary: "bash"
args:
- .evergreen/scripts/run-gcpkms-fail-test.sh
- name: testazurekms-task
commands:
@ -1736,18 +1635,15 @@ tasks:
- name: "check-import-time"
tags: ["pr"]
commands:
- command: shell.exec
- command: subprocess.exec
type: test
params:
shell: "bash"
binary: bash
working_dir: src
script: |
. .evergreen/scripts/env.sh
set -x
export BASE_SHA=${revision}
export HEAD_SHA=${github_commit}
bash .evergreen/run-import-time-test.sh
args:
- .evergreen/scripts/check-import-time.sh
- ${revision}
- ${github_commit}
- name: "backport-pr"
allowed_requesters: ["commit"]
commands:

2
.evergreen/hatch.sh
View File

@ -29,7 +29,7 @@ else # Set up virtualenv before installing hatch
# Ensure hatch does not write to user or global locations.
touch hatch_config.toml
HATCH_CONFIG=$(pwd)/hatch_config.toml
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
fi
export HATCH_CONFIG

8
.evergreen/run-tests.sh
View File

@ -53,18 +53,18 @@ if [ -z "${NO_EXT:-}" ]; then
fi
if [ "$AUTH" != "noauth" ]; then
if [ ! -z "$TEST_DATA_LAKE" ]; then
if [ -n "$TEST_DATA_LAKE" ]; then
export DB_USER="mhuser"
export DB_PASSWORD="pencil"
elif [ ! -z "$TEST_SERVERLESS" ]; then
source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh
elif [ -n "$TEST_SERVERLESS" ]; then
source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
export DB_USER=$SERVERLESS_ATLAS_USER
export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
export MONGODB_URI="$SERVERLESS_URI"
echo "MONGODB_URI=$MONGODB_URI"
export SINGLE_MONGOS_LB_URI=$MONGODB_URI
export MULTI_MONGOS_LB_URI=$MONGODB_URI
elif [ ! -z "$TEST_AUTH_OIDC" ]; then
elif [ -n "$TEST_AUTH_OIDC" ]; then
export DB_USER=$OIDC_ADMIN_USER
export DB_PASSWORD=$OIDC_ADMIN_PWD
export DB_IP="$MONGODB_URI"

8
.evergreen/scripts/archive-mongodb-logs.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -o xtrace
mkdir out_dir
# shellcheck disable=SC2156
find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
tar zcvf mongodb-logs.tar.gz -C out_dir/ .
rm -rf out_dir

46
.evergreen/scripts/bootstrap-mongo-orchestration.sh Normal file
View File

@ -0,0 +1,46 @@
#!/bin/bash
set -o xtrace
# Enable core dumps if enabled on the machine
# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
if [ -f /proc/self/coredump_filter ]; then
# Set the shell process (and its children processes) to dump ELF headers (bit 4),
# anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
echo 0x13 >/proc/self/coredump_filter
if [ -f /sbin/sysctl ]; then
# Check that the core pattern is set explicitly on our distro image instead
# of being the OS's default value. This ensures that coredump names are consistent
# across distros and can be picked up by Evergreen.
core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
if [ "$core_pattern" = "dump_%e.%p.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
fi
if [ "$(uname -s)" = "Darwin" ]; then
core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
if [ -n "${skip_crypt_shared}" ]; then
export SKIP_CRYPT_SHARED=1
fi
MONGODB_VERSION=${VERSION} \
TOPOLOGY=${TOPOLOGY} \
AUTH=${AUTH:-noauth} \
SSL=${SSL:-nossl} \
STORAGE_ENGINE=${STORAGE_ENGINE:-} \
DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \
ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \
REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-} \
LOAD_BALANCER=${LOAD_BALANCER:-} \
bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
# run-orchestration generates expansion file with the MONGODB_URI for the cluster

7
.evergreen/scripts/check-import-time.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
. .evergreen/scripts/env.sh
set -x
export BASE_SHA="$1"
export HEAD_SHA="$2"
bash .evergreen/run-import-time-test.sh

7
.evergreen/scripts/cleanup.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
. .evergreen/hatch.sh encryption:teardown
fi
rm -rf "${DRIVERS_TOOLS}" || true
rm -f ./secrets-export.sh || true

19
.evergreen/scripts/configure-env.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash -ex
#!/bin/bash -eux
# Get the current unique version of this checkout
# shellcheck disable=SC2154
@ -29,7 +29,7 @@ fi
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
cat <<EOT > $SCRIPT_DIR/env.sh
cat <<EOT > "$SCRIPT_DIR"/env.sh
set -o errexit
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export CURRENT_VERSION="$CURRENT_VERSION"
@ -38,6 +38,21 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS"
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}"
export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}"
export MONGODB_STARTED="${MONGODB_STARTED:-}"
export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}"
export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}"
export NO_EXT="${NO_EXT:-}"
export COVERAGE="${COVERAGE:-}"
export COMPRESSORS="${COMPRESSORS:-}"
export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}"
export SKIP_HATCH="${SKIP_HATCH:-}"
export skip_crypt_shared="${skip_crypt_shared:-}"
export STORAGE_ENGINE="${STORAGE_ENGINE:-}"
export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}"
export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export PATH="$MONGODB_BINARIES:$PATH"

4
.evergreen/scripts/download-and-merge-coverage.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
# Download all the task coverage files.
aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/

8
.evergreen/scripts/fix-absolute-paths.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for filename in $(find $DRIVERS_TOOLS -name \*.json); do
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
done

5
.evergreen/scripts/init-test-results.sh Normal file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json

6
.evergreen/scripts/install-dependencies.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
set -o xtrace
file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh"
# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
[ -f "$file" ] && bash "$file" || echo "$file not available, skipping"

8
.evergreen/scripts/make-files-executable.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
chmod +x "$i"
done

12
.evergreen/scripts/prepare-resources.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/bash
. src/.evergreen/scripts/env.sh
set -o xtrace
rm -rf $DRIVERS_TOOLS
if [ "$PROJECT" = "drivers-tools" ]; then
# If this was a patch build, doing a fresh clone would not actually test the patch
cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS
else
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
fi
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config

7
.evergreen/scripts/run-atlas-tests.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# Disable xtrace for security reasons (just in case it was accidentally set).
set +x
set -o errexit
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

15
.evergreen/scripts/run-aws-ecs-auth-test.sh Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
# shellcheck disable=SC2154
if [ "${skip_ECS_auth_test}" = "true" ]; then
echo "This platform does not support the ECS auth test, skipping..."
exit 0
fi
set -ex
cd "$DRIVERS_TOOLS"/.evergreen/auth_aws
. ./activate-authawsvenv.sh
. aws_setup.sh ecs
export MONGODB_BINARIES="$MONGODB_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
python aws_tester.py ecs
cd -

4
.evergreen/scripts/run-doctests.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
set -o xtrace
PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test

6
.evergreen/scripts/run-enterprise-auth-tests.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
# Disable xtrace for security reasons (just in case it was accidentally set).
set +x
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

7
.evergreen/scripts/run-gcpkms-fail-test.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
. .evergreen/scripts/env.sh
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg

22
.evergreen/scripts/run-getdata.sh Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -o xtrace
. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true
get_distro || true
echo $DISTRO
echo $MARCH
echo $OS
uname -a || true
ls /etc/*release* || true
cc --version || true
gcc --version || true
clang --version || true
gcov --version || true
lcov --version || true
llvm-cov --version || true
echo $PATH
ls -la /usr/local/Cellar/llvm/*/bin/ || true
ls -la /usr/local/Cellar/ || true
scan-build --version || true
genhtml --version || true
valgrind --version || true

3
.evergreen/scripts/run-load-balancer.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start

5
.evergreen/scripts/run-mockupdb-tests.sh Normal file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -o xtrace
export PYTHON_BINARY=${PYTHON_BINARY}
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb

2
.evergreen/run-mod-wsgi-tests.sh → .evergreen/scripts/run-mod-wsgi-tests.sh
View File

@ -28,7 +28,7 @@ export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_
export PYTHONHOME=/opt/python/$PYTHON_VERSION
# If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead
# of daemon mode (WSGIDaemonProcess).
if [ -n "$MOD_WSGI_EMBEDDED" ]; then
if [ -n "${MOD_WSGI_EMBEDDED:-}" ]; then
export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf
else
export MOD_WSGI_CONF=mod_wsgi_test.conf

10
.evergreen/run-mongodb-aws-test.sh → .evergreen/scripts/run-mongodb-aws-test.sh
View File

@ -13,10 +13,16 @@ set -o errexit # Exit the script with error if any of the commands fail
# mechanism.
# PYTHON_BINARY The Python version to use.
echo "Running MONGODB-AWS authentication tests"
# shellcheck disable=SC2154
if [ "${skip_EC2_auth_test:-}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then
echo "This platform does not support the EC2 auth test, skipping..."
exit 0
fi
echo "Running MONGODB-AWS authentication tests for $1"
# Handle credentials and environment setup.
. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
# show test output
set -x

8
.evergreen/scripts/run-ocsp-test.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
TEST_OCSP=1 \
PYTHON_BINARY="${PYTHON_BINARY}" \
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

4
.evergreen/scripts/run-perf-tests.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
PROJECT_DIRECTORY=${PROJECT_DIRECTORY}
bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh

55
.evergreen/scripts/run-tests.sh Normal file
View File

@ -0,0 +1,55 @@
#!/bin/bash
# Disable xtrace
set +x
if [ -n "${MONGODB_STARTED}" ]; then
export PYMONGO_MUST_CONNECT=true
fi
if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
export PYMONGO_DISABLE_TEST_COMMANDS=1
fi
if [ -n "${test_encryption}" ]; then
# Disable xtrace (just in case it was accidentally set).
set +x
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
export TEST_ENCRYPTION=1
if [ -n "${test_encryption_pyopenssl}" ]; then
export TEST_ENCRYPTION_PYOPENSSL=1
fi
fi
if [ -n "${test_crypt_shared}" ]; then
export TEST_CRYPT_SHARED=1
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
fi
if [ -n "${test_pyopenssl}" ]; then
export TEST_PYOPENSSL=1
fi
if [ -n "${SETDEFAULTENCODING}" ]; then
export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
fi
if [ -n "${test_loadbalancer}" ]; then
export TEST_LOADBALANCER=1
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
fi
if [ -n "${test_serverless}" ]; then
export TEST_SERVERLESS=1
fi
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
export TEST_INDEX_MANAGEMENT=1
fi
if [ -n "${SKIP_CSOT_TESTS}" ]; then
export SKIP_CSOT_TESTS=1
fi
GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
PYTHON_BINARY=${PYTHON_BINARY} \
NO_EXT=${NO_EXT} \
COVERAGE=${COVERAGE} \
COMPRESSORS=${COMPRESSORS} \
AUTH=${AUTH} \
SSL=${SSL} \
TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
TEST_SUITES=${TEST_SUITES:-} \
MONGODB_API_VERSION=${MONGODB_API_VERSION} \
SKIP_HATCH=${SKIP_HATCH} \
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

21
.evergreen/scripts/run-with-env.sh Normal file
View File

@ -0,0 +1,21 @@
#!/bin/bash -eu
# Example use: bash run-with-env.sh run-tests.sh {args...}
# Parameter expansion to get just the current directory's name
if [ "${PWD##*/}" == "src" ]; then
. .evergreen/scripts/env.sh
if [ -f ".evergreen/scripts/test-env.sh" ]; then
. .evergreen/scripts/test-env.sh
fi
else
. src/.evergreen/scripts/env.sh
if [ -f "src/.evergreen/scripts/test-env.sh" ]; then
. src/.evergreen/scripts/test-env.sh
fi
fi
set -eu
# shellcheck source=/dev/null
. "$@"

5
.evergreen/scripts/setup-encryption.sh Normal file
View File

@ -0,0 +1,5 @@
#!/bin/bash
if [ -n "${test_encryption}" ]; then
./.evergreen/hatch.sh encryption:setup
fi

27
.evergreen/scripts/setup-tests.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash -eux
PROJECT_DIRECTORY="$(pwd)"
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
if [ -f "$SCRIPT_DIR/test-env.sh" ]; then
echo "Reading $SCRIPT_DIR/test-env.sh file"
. "$SCRIPT_DIR/test-env.sh"
exit 0
fi
cat <<EOT > "$SCRIPT_DIR"/test-env.sh
export test_encryption="${test_encryption:-}"
export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}"
export test_crypt_shared="${test_crypt_shared:-}"
export test_pyopenssl="${test_pyopenssl:-}"
export test_loadbalancer="${test_loadbalancer:-}"
export test_serverless="${test_serverless:-}"
export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}"
export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}"
export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}"
export AUTH="${AUTH:-noauth}"
export SSL="${SSL:-nossl}"
export PYTHON_BINARY="${PYTHON_BINARY:-}"
EOT
chmod +x "$SCRIPT_DIR"/test-env.sh

5
.evergreen/scripts/stop-load-balancer.sh Normal file
View File

@ -0,0 +1,5 @@
#!/bin/bash
cd "${DRIVERS_TOOLS}"/.evergreen || exit
DRIVERS_TOOLS=${DRIVERS_TOOLS}
bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop

7
.evergreen/scripts/teardown-aws.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || exit
if [ -f "./aws_e2e_setup.json" ]; then
. ./activate-authawsvenv.sh
python ./lib/aws_assign_instance_profile.py
fi

7
.evergreen/scripts/teardown-docker.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# Remove all Docker images
DOCKER=$(command -v docker) || true
if [ -n "$DOCKER" ]; then
docker rmi -f "$(docker images -a -q)" &> /dev/null || true
fi

3
.evergreen/scripts/upload-coverage-report.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
aws s3 cp htmlcov/ s3://"$1"/coverage/"$2"/"$3"/htmlcov/ --recursive --acl public-read --region us-east-1

11
.evergreen/scripts/windows-fix.sh Normal file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set +x
. src/.evergreen/scripts/env.sh
# shellcheck disable=SC2044
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
< "$i" tr -d '\r' >"$i".new
mv "$i".new "$i"
done
# Copy client certificate because symlinks do not work on Windows.
cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem

7
.evergreen/setup-encryption.sh
View File

@ -52,6 +52,9 @@ ls -la libmongocrypt
ls -la libmongocrypt/nocrypto
if [ -z "${SKIP_SERVERS:-}" ]; then
bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh
bash ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh
PYTHON_BINARY_OLD=${PYTHON_BINARY}
export PYTHON_BINARY=""
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh
export PYTHON_BINARY=$PYTHON_BINARY_OLD
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh
fi

4
.evergreen/utils.sh
View File

@ -17,7 +17,7 @@ find_python3() {
elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.9" ]; then
PYTHON="/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3"
fi
elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
PYTHON="C:/python/Python39/python.exe"
else
# Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.9+.
@ -56,7 +56,7 @@ createvirtualenv () {
# Workaround for bug in older versions of virtualenv.
$VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH
fi
if [ "Windows_NT" = "$OS" ]; then
if [ "Windows_NT" = "${OS:-}" ]; then
# Workaround https://bugs.python.org/issue32451:
# mongovenv/Scripts/activate: line 3: $'\r': command not found
dos2unix $VENVPATH/Scripts/activate || true

8
test/asynchronous/test_client_context.py
View File

@ -25,7 +25,7 @@ _IS_SYNC = False
class TestAsyncClientContext(AsyncUnitTest):
def test_must_connect(self):
if "PYMONGO_MUST_CONNECT" not in os.environ:
if not os.environ.get("PYMONGO_MUST_CONNECT"):
raise SkipTest("PYMONGO_MUST_CONNECT is not set")
self.assertTrue(
@ -37,7 +37,7 @@ class TestAsyncClientContext(AsyncUnitTest):
)
def test_serverless(self):
if "TEST_SERVERLESS" not in os.environ:
if not os.environ.get("TEST_SERVERLESS"):
raise SkipTest("TEST_SERVERLESS is not set")
self.assertTrue(
@ -47,7 +47,7 @@ class TestAsyncClientContext(AsyncUnitTest):
)
def test_enableTestCommands_is_disabled(self):
if "PYMONGO_DISABLE_TEST_COMMANDS" not in os.environ:
if not os.environ.get("PYMONGO_DISABLE_TEST_COMMANDS"):
raise SkipTest("PYMONGO_DISABLE_TEST_COMMANDS is not set")
self.assertFalse(
@ -56,7 +56,7 @@ class TestAsyncClientContext(AsyncUnitTest):
)
def test_setdefaultencoding_worked(self):
if "SETDEFAULTENCODING" not in os.environ:
if not os.environ.get("SETDEFAULTENCODING"):
raise SkipTest("SETDEFAULTENCODING is not set")
self.assertEqual(sys.getdefaultencoding(), os.environ["SETDEFAULTENCODING"])

2
test/mod_wsgi_test/README.rst
View File

@ -107,4 +107,4 @@ Automation
At MongoDB, Inc. we use a continuous integration job that tests each
combination in the matrix. The job starts up Apache, starts a single server
or replica set, and runs ``test_client.py`` with the proper arguments.
See `run-mod-wsgi-tests.sh <https://github.com/mongodb/mongo-python-driver/blob/master/.evergreen/run-mod-wsgi-tests.sh>`_
See `run-mod-wsgi-tests.sh <https://github.com/mongodb/mongo-python-driver/blob/master/.evergreen/scripts/run-mod-wsgi-tests.sh>`_

8
test/test_client_context.py
View File

@ -25,7 +25,7 @@ _IS_SYNC = True
class TestClientContext(UnitTest):
def test_must_connect(self):
if "PYMONGO_MUST_CONNECT" not in os.environ:
if not os.environ.get("PYMONGO_MUST_CONNECT"):
raise SkipTest("PYMONGO_MUST_CONNECT is not set")
self.assertTrue(
@ -37,7 +37,7 @@ class TestClientContext(UnitTest):
)
def test_serverless(self):
if "TEST_SERVERLESS" not in os.environ:
if not os.environ.get("TEST_SERVERLESS"):
raise SkipTest("TEST_SERVERLESS is not set")
self.assertTrue(
@ -47,7 +47,7 @@ class TestClientContext(UnitTest):
)
def test_enableTestCommands_is_disabled(self):
if "PYMONGO_DISABLE_TEST_COMMANDS" not in os.environ:
if not os.environ.get("PYMONGO_DISABLE_TEST_COMMANDS"):
raise SkipTest("PYMONGO_DISABLE_TEST_COMMANDS is not set")
self.assertFalse(
@ -56,7 +56,7 @@ class TestClientContext(UnitTest):
)
def test_setdefaultencoding_worked(self):
if "SETDEFAULTENCODING" not in os.environ:
if not os.environ.get("SETDEFAULTENCODING"):
raise SkipTest("SETDEFAULTENCODING is not set")
self.assertEqual(sys.getdefaultencoding(), os.environ["SETDEFAULTENCODING"])