From fc220532df094efdd24bd4024e83d45caf33df50 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 17 Nov 2023 10:35:46 -0800 Subject: [PATCH] PYTHON-3823 Merge perf task into main project (#1436) --- .evergreen/config.yml | 37 ++++++ .evergreen/perf.yml | 220 ---------------------------------- .evergreen/run-perf-tests.sh | 8 +- test/performance/perf_test.py | 7 +- 4 files changed, 45 insertions(+), 227 deletions(-) delete mode 100644 .evergreen/perf.yml diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b3ea32310..46adaf4e7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -905,6 +905,25 @@ functions: content_type: ${content_type|application/gzip} display_name: Release files all + "run perf tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh + + "attach benchmark test results": + - command: attach.results + params: + file_location: src/report.json + + "send dashboard data": + - command: perf.send + params: + file: src/results.json + pre: - func: "fetch source" - func: "prepare resources" @@ -2154,6 +2173,17 @@ tasks: SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ ./.evergreen/tox.sh -m test-eg + - name: "perf-6.0-standalone" + tags: ["perf"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "v6.0-perf" + TOPOLOGY: "server" + - func: "run perf tests" + - func: "attach benchmark test results" + - func: "send dashboard data" + axes: # Choice of distro - id: platform @@ -3117,6 +3147,13 @@ buildvariants: tasks: - ".release_tag" +- name: "perf-tests" + display_name: "Performance Benchmark Tests" + batchtime: 10080 # 7 days + run_on: rhel90-dbx-perf-large + tasks: + - name: "perf-6.0-standalone" + # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available # Debian 8.1 only supports MongoDB 3.4+ diff --git a/.evergreen/perf.yml b/.evergreen/perf.yml deleted file mode 100644 index e9a193cd1..000000000 --- a/.evergreen/perf.yml +++ /dev/null @@ -1,220 +0,0 @@ -######################################## -# Evergreen Template for MongoDB Drivers -######################################## - -# When a task that used to pass starts to fail -# Go through all versions that may have been skipped to detect -# when the task started failing -stepback: true - -# Mark a failure as a system/bootstrap failure (purple box) rather then a task -# failure by default. -# Actual testing tasks are marked with `type: test` -command_type: system - -# Protect ourself against rogue test case, or curl gone wild, that runs forever -# Good rule of thumb: the averageish length a task takes, times 5 -# That roughly accounts for variable system performance for various buildvariants -exec_timeout_secs: 3600 # 60 minutes is the longest we'll ever run - -# What to do when evergreen hits the timeout (`post:` tasks are run automatically) -timeout: - - command: shell.exec - params: - script: | - ls -la - -functions: - "fetch source": - # Executes clone and applies the submitted patch, if any - - command: git.get_project - params: - directory: "src" - # Applies the subitted patch, if any - # Deprecated. Should be removed. But still needed for certain agents (ZAP) - - command: git.apply_patch - # Make an evergreen exapanstion file with dynamic values - - command: shell.exec - params: - working_dir: "src" - script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi - - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" - export PROJECT_DIRECTORY="$(pwd)" - - # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory - if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) - fi - - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - PREPARE_SHELL: | - set -o errexit - export SKIP_LEGACY_SHELL=1 - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" - EOT - # See what we've done - cat expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: src/expansion.yml - - "prepare resources": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - rm -rf $DRIVERS_TOOLS - if [ "${project}" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - - "bootstrap mongo-orchestration": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with the MONGODB_URI for the cluster - - command: expansions.update - params: - file: mo-expansion.yml - - "stop mongo-orchestration": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh - - "run perf tests": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh - - "attach benchmark test results": - - command: attach.results - params: - file_location: src/report.json - - "send dashboard data": - - command: perf.send - params: - file: src/results.json - - "cleanup": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - rm -rf $DRIVERS_TOOLS || true - - "fix absolute paths": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done - - "windows fix": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem - - "make files executable": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - chmod +x $i - done - - "install dependencies": - - command: shell.exec - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" - # Don't use ${file} syntax here because evergreen treats it as an empty expansion. - [ -f "$file" ] && bash $file || echo "$file not available, skipping" - -pre: - - func: "fetch source" - - func: "prepare resources" - # We don't run perf on Windows (yet) - #- func: "windows fix" - - func: "fix absolute paths" - - func: "make files executable" - # We're not testing with TLS (yet) - #- func: "install dependencies" - -post: - - func: "stop mongo-orchestration" - - func: "cleanup" - -tasks: - - name: "perf-6.0-standalone" - tags: ["perf"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "v6.0-perf" - TOPOLOGY: "server" - - func: "run perf tests" - - func: "attach benchmark test results" - - func: "send dashboard data" - -buildvariants: - -- name: "perf-tests" - display_name: "Performance Benchmark Tests" - batchtime: 10080 # 7 days - run_on: rhel90-dbx-perf-large - tasks: - - name: "perf-6.0-standalone" diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index 6363f4e43..9d6afacf7 100644 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -3,14 +3,14 @@ set -o xtrace set -o errexit -git clone https://github.com/mongodb-labs/driver-performance-test-data.git -cd driver-performance-test-data +git clone --depth 1 https://github.com/mongodb/specifications.git +pushd specifications/source/benchmarking/data tar xf extended_bson.tgz tar xf parallel.tgz tar xf single_and_multi_document.tgz -cd .. +popd -export TEST_PATH="${PROJECT_DIRECTORY}/driver-performance-test-data" +export TEST_PATH="${PROJECT_DIRECTORY}/specifications/source/benchmarking/data" export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 diff --git a/test/performance/perf_test.py b/test/performance/perf_test.py index 109fff3c1..6aabb595e 100644 --- a/test/performance/perf_test.py +++ b/test/performance/perf_test.py @@ -83,9 +83,10 @@ class PerformanceTest: pass def tearDown(self): - name = self.__class__.__name__ + # Remove "Test" so that TestFlatEncoding is reported as "FlatEncoding". + name = self.__class__.__name__[4:] median = self.percentile(50) - bytes_per_sec = self.data_size / median + megabytes_per_sec = self.data_size / median / 1000000 print(f"Running {self.__class__.__name__}. MEDIAN={self.percentile(50)}") result_data.append( { @@ -96,7 +97,7 @@ class PerformanceTest: }, }, "metrics": [ - {"name": "bytes_per_sec", "value": bytes_per_sec}, + {"name": "megabytes_per_sec", "type": "MEDIAN", "value": megabytes_per_sec}, ], } )