mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-4631: Pushed PREPARE_SHELL creation into an env.sh file (#1788)

Browse Source 
Co-authored-by: Steven Silvester <steve.silvester@mongodb.com>
This commit is contained in:
Jib 2024-09-05 09:39:55 -04:00 committed by GitHub
parent 4d4813070d
commit 26c55048d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 106 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
.evergreen/config.yml
View File

@ -34,51 +34,14 @@ functions:
# Applies the subitted patch, if any
# Deprecated. Should be removed. But still needed for certain agents (ZAP)
- command: git.apply_patch
# Make an evergreen exapanstion file with dynamic values
- command: shell.exec
# Make an evergreen expansion file with dynamic values
- command: subprocess.exec
params:
include_expansions_in_env: ["is_patch", "project", "version_id"]
binary: bash
working_dir: "src"
script: |
set +x
# Get the current unique version of this checkout
if [ "${is_patch}" = "true" ]; then
CURRENT_VERSION=$(git describe)-patch-${version_id}
else
CURRENT_VERSION=latest
fi
export DRIVERS_TOOLS="$(dirname $(pwd))/drivers-tools"
export PROJECT_DIRECTORY="$(pwd)"
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
fi
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
cat <<EOT > expansion.yml
CURRENT_VERSION: "$CURRENT_VERSION"
DRIVERS_TOOLS: "$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME"
MONGODB_BINARIES: "$MONGODB_BINARIES"
PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
PREPARE_SHELL: |
set -o errexit
export SKIP_LEGACY_SHELL=1
export DRIVERS_TOOLS="$DRIVERS_TOOLS"
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export PATH="$MONGODB_BINARIES:$PATH"
export PROJECT="${project}"
export PIP_QUIET=1
EOT
args:
- .evergreen/scripts/configure-env.sh
# Load the expansion file to make an evergreen variable with the current unique version
- command: expansions.update
params:
@ -88,14 +51,14 @@ functions:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
set -o xtrace
rm -rf $DRIVERS_TOOLS
if [ "${project}" = "drivers-tools" ]; then
if [ "$PROJECT" = "drivers-tools" ]; then
# If this was a patch build, doing a fresh clone would not actually test the patch
cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS
cp -R ${PROJECT_DIRECTORY}/ ${DRIVERS_TOOLS}
else
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
fi
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
@ -129,12 +92,12 @@ functions:
script: |
# Download all the task coverage files.
aws s3 cp --recursive s3://${bucket_name}/coverage/${revision}/${version_id}/coverage/ coverage/
- command: shell.exec
- command: subprocess.exec
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
bash .evergreen/combine-coverage.sh
binary: bash
args:
- .evergreen/combine-coverage.sh
# Upload the resulting html coverage report.
- command: shell.exec
params:
@ -164,7 +127,7 @@ functions:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
set -o xtrace
mkdir out_dir
find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
@ -266,7 +229,7 @@ functions:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
set -o xtrace
# Enable core dumps if enabled on the machine
@ -325,13 +288,13 @@ functions:
type: setup
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
- command: shell.exec
type: setup
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
sleep 1
docker ps
@ -340,7 +303,7 @@ functions:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
set -o xtrace
bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
@ -350,7 +313,7 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -o xtrace
PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} \
MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
@ -362,7 +325,7 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -o xtrace
export PYTHON_BINARY=${PYTHON_BINARY}
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb
@ -373,7 +336,7 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -o xtrace
PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test
@ -385,7 +348,7 @@ functions:
background: true
include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
if [ -n "${test_encryption}" ]; then
./.evergreen/hatch.sh encryption:setup
fi
@ -397,7 +360,7 @@ functions:
script: |
# Disable xtrace
set +x
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
if [ -n "${MONGODB_STARTED}" ]; then
export PYMONGO_MUST_CONNECT=true
fi
@ -497,7 +460,7 @@ functions:
shell: "bash"
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh regular
"run aws auth test with assume role credentials":
@ -507,7 +470,7 @@ functions:
shell: "bash"
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh assume-role
"run aws auth test with aws EC2 credentials":
@ -521,7 +484,7 @@ functions:
echo "This platform does not support the EC2 auth test, skipping..."
exit 0
fi
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh ec2
"run aws auth test with aws web identity credentials":
@ -535,7 +498,7 @@ functions:
echo "This platform does not support the web identity auth test, skipping..."
exit 0
fi
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
# Test with and without AWS_ROLE_SESSION_NAME set.
.evergreen/run-mongodb-aws-test.sh web-identity
AWS_ROLE_SESSION_NAME="test" \
@ -558,7 +521,7 @@ functions:
working_dir: "src"
shell: bash
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh env-creds
"run aws auth test with aws credentials and session token as environment variables":
@ -568,7 +531,7 @@ functions:
working_dir: "src"
shell: bash
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
.evergreen/run-mongodb-aws-test.sh session-creds
"run aws ECS auth test":
@ -582,12 +545,12 @@ functions:
echo "This platform does not support the ECS auth test, skipping..."
exit 0
fi
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -ex
cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
. ./activate-authawsvenv.sh
. aws_setup.sh ecs
export MONGODB_BINARIES="${MONGODB_BINARIES}";
export MONGODB_BINARIES="$MONGODB_BINARIES";
export PROJECT_DIRECTORY="${PROJECT_DIRECTORY}";
python aws_tester.py ecs
cd -
@ -597,9 +560,9 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
. .evergreen/hatch.sh encryption:teardown
rm -rf $DRIVERS_TOOLS || true
rm -rf ${DRIVERS_TOOLS} || true
rm -f ./secrets-export.sh || true
"fix absolute paths":
@ -607,7 +570,7 @@ functions:
params:
script: |
set +x
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename
done
@ -617,20 +580,20 @@ functions:
params:
script: |
set +x
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
cat $i | tr -d '\r' > $i.new
mv $i.new $i
done
# Copy client certificate because symlinks do not work on Windows.
cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem
cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem
"make files executable":
- command: shell.exec
params:
script: |
set +x
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
chmod +x $i
done
@ -640,7 +603,7 @@ functions:
params:
script: |
set +x
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json
"install dependencies":
@ -648,7 +611,7 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -o xtrace
file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh"
# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
@ -679,10 +642,10 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
TEST_OCSP=1 \
PYTHON_BINARY=${PYTHON_BINARY} \
CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
@ -691,7 +654,7 @@ functions:
params:
background: true
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd ${DRIVERS_TOOLS}/.evergreen/ocsp
. ./activate-ocspvenv.sh
python ocsp_mock.py \
@ -704,7 +667,7 @@ functions:
params:
background: true
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd ${DRIVERS_TOOLS}/.evergreen/ocsp
. ./activate-ocspvenv.sh
python ocsp_mock.py \
@ -719,7 +682,7 @@ functions:
params:
background: true
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd ${DRIVERS_TOOLS}/.evergreen/ocsp
. ./activate-ocspvenv.sh
python ocsp_mock.py \
@ -732,7 +695,7 @@ functions:
params:
background: true
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd ${DRIVERS_TOOLS}/.evergreen/ocsp
. ./activate-ocspvenv.sh
python ocsp_mock.py \
@ -774,7 +737,7 @@ functions:
params:
shell: "bash"
script: |
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd "${DRIVERS_TOOLS}/.evergreen/auth_aws"
if [ -f "./aws_e2e_setup.json" ]; then
. ./activate-authawsvenv.sh
@ -794,7 +757,7 @@ functions:
params:
working_dir: "src"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh
"attach benchmark test results":
@ -1889,7 +1852,7 @@ tasks:
shell: bash
script: |-
set -o errexit
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd src
git add .
git commit -m "add files"
@ -1906,7 +1869,7 @@ tasks:
shell: bash
script: |-
set -o errexit
${PREPARE_SHELL}
. src/.evergreen/scripts/env.sh
cd src
git add .
git commit -m "add files"
@ -1974,7 +1937,7 @@ tasks:
working_dir: "src"
shell: "bash"
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
@ -2051,7 +2014,7 @@ tasks:
shell: "bash"
working_dir: src
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -x
export CONFIG=$PROJECT_DIRECTORY/.github/reviewers.txt
export SCRIPT="$DRIVERS_TOOLS/.evergreen/github_app/assign-reviewer.sh"
@ -2067,7 +2030,7 @@ tasks:
shell: "bash"
working_dir: src
script: |
${PREPARE_SHELL}
. .evergreen/scripts/env.sh
set -x
export BASE_SHA=${revision}
export HEAD_SHA=${github_commit}

53
.evergreen/scripts/configure-env.sh Normal file
View File

@ -0,0 +1,53 @@
#!/bin/bash -ex
# Get the current unique version of this checkout
# shellcheck disable=SC2154
if [ "$is_patch" = "true" ]; then
# shellcheck disable=SC2154
CURRENT_VERSION="$(git describe)-patch-$version_id"
else
CURRENT_VERSION=latest
fi
PROJECT_DIRECTORY="$(pwd)"
DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
fi
SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
if [ -f "$SCRIPT_DIR/env.sh" ]; then
echo "Reading $SCRIPT_DIR/env.sh file"
. "$SCRIPT_DIR/env.sh"
exit 0
fi
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
cat <<EOT > $SCRIPT_DIR/env.sh
set -o errexit
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export CURRENT_VERSION="$CURRENT_VERSION"
export SKIP_LEGACY_SHELL=1
export DRIVERS_TOOLS="$DRIVERS_TOOLS"
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export PATH="$MONGODB_BINARIES:$PATH"
# shellcheck disable=SC2154
export PROJECT="$project"
export PIP_QUIET=1
EOT
# Add these expansions to make it easier to call out tests scripts from the EVG yaml
cat <<EOT > expansion.yml
DRIVERS_TOOLS: "$DRIVERS_TOOLS"
PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
EOT