PYTHON-3823 Merge perf task into main project (#1436)

2023-11-17 10:35:46 -08:00 · 2023-11-17 10:35:46 -08:00 · fc220532df
commit fc220532df
parent 2ac7f0d88d
4 changed files with 45 additions and 227 deletions
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@ -905,6 +905,25 @@ functions:
        content_type: ${content_type|application/gzip}
        display_name: Release files all

+  "run perf tests":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh
+
+  "attach benchmark test results":
+    - command: attach.results
+      params:
+        file_location: src/report.json
+
+  "send dashboard data":
+    - command: perf.send
+      params:
+        file: src/results.json
+
 pre:
  - func: "fetch source"
  - func: "prepare resources"
@ -2154,6 +2173,17 @@ tasks:
            SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
            ./.evergreen/tox.sh -m test-eg

+    - name: "perf-6.0-standalone"
+      tags: ["perf"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "v6.0-perf"
+            TOPOLOGY: "server"
+        - func: "run perf tests"
+        - func: "attach benchmark test results"
+        - func: "send dashboard data"
+
 axes:
  # Choice of distro
  - id: platform
@ -3117,6 +3147,13 @@ buildvariants:
  tasks:
  - ".release_tag"

+- name: "perf-tests"
+  display_name: "Performance Benchmark Tests"
+  batchtime: 10080  # 7 days
+  run_on: rhel90-dbx-perf-large
+  tasks:
+     - name: "perf-6.0-standalone"
+
      # Platform notes
      # i386 builds of OpenSSL or Cyrus SASL are not available
      # Debian 8.1 only supports MongoDB 3.4+
--- a/.evergreen/perf.yml
+++ b/.evergreen/perf.yml
@ -1,220 +0,0 @@
-########################################
-# Evergreen Template for MongoDB Drivers
-########################################
-
-# When a task that used to pass starts to fail
-# Go through all versions that may have been skipped to detect
-# when the task started failing
-stepback: true
-
-# Mark a failure as a system/bootstrap failure (purple box) rather then a task
-# failure by default.
-# Actual testing tasks are marked with `type: test`
-command_type: system
-
-# Protect ourself against rogue test case, or curl gone wild, that runs forever
-# Good rule of thumb: the averageish length a task takes, times 5
-# That roughly accounts for variable system performance for various buildvariants
-exec_timeout_secs: 3600 # 60 minutes is the longest we'll ever run
-
-# What to do when evergreen hits the timeout (`post:` tasks are run automatically)
-timeout:
-  - command: shell.exec
-    params:
-      script: |
-        ls -la
-
-functions:
-  "fetch source":
-    # Executes clone and applies the submitted patch, if any
-    - command: git.get_project
-      params:
-        directory: "src"
-    # Applies the subitted patch, if any
-    # Deprecated. Should be removed. But still needed for certain agents (ZAP)
-    - command: git.apply_patch
-    # Make an evergreen exapanstion file with dynamic values
-    - command: shell.exec
-      params:
-        working_dir: "src"
-        script: |
-           # Get the current unique version of this checkout
-           if [ "${is_patch}" = "true" ]; then
-              CURRENT_VERSION=$(git describe)-patch-${version_id}
-           else
-              CURRENT_VERSION=latest
-           fi
-
-           export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
-           export PROJECT_DIRECTORY="$(pwd)"
-
-           # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
-           if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-              export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
-              export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
-           fi
-
-           export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
-           export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
-           export UPLOAD_BUCKET="${project}"
-
-           cat <<EOT > expansion.yml
-           CURRENT_VERSION: "$CURRENT_VERSION"
-           DRIVERS_TOOLS: "$DRIVERS_TOOLS"
-           MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME"
-           MONGODB_BINARIES: "$MONGODB_BINARIES"
-           UPLOAD_BUCKET: "$UPLOAD_BUCKET"
-           PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
-           PREPARE_SHELL: |
-              set -o errexit
-              export SKIP_LEGACY_SHELL=1
-              export DRIVERS_TOOLS="$DRIVERS_TOOLS"
-              export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
-              export MONGODB_BINARIES="$MONGODB_BINARIES"
-              export UPLOAD_BUCKET="$UPLOAD_BUCKET"
-              export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
-
-              export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-              export PATH="$MONGODB_BINARIES:$PATH"
-              export PROJECT="${project}"
-           EOT
-           # See what we've done
-           cat expansion.yml
-
-    # Load the expansion file to make an evergreen variable with the current unique version
-    - command: expansions.update
-      params:
-        file: src/expansion.yml
-
-  "prepare resources":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          rm -rf $DRIVERS_TOOLS
-          if [ "${project}" = "drivers-tools" ]; then
-            # If this was a patch build, doing a fresh clone would not actually test the patch
-            cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS
-          else
-            git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
-          fi
-          echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
-
-  "bootstrap mongo-orchestration":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
-    # run-orchestration generates expansion file with the MONGODB_URI for the cluster
-    - command: expansions.update
-      params:
-        file: mo-expansion.yml
-
-  "stop mongo-orchestration":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
-
-  "run perf tests":
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: "src"
-        script: |
-          ${PREPARE_SHELL}
-          PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh
-
-  "attach benchmark test results":
-    - command: attach.results
-      params:
-        file_location: src/report.json
-
-  "send dashboard data":
-    - command: perf.send
-      params:
-        file: src/results.json
-
-  "cleanup":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          rm -rf $DRIVERS_TOOLS || true
-
-  "fix absolute paths":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do
-            perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename
-          done
-
-  "windows fix":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
-            cat $i | tr -d '\r' > $i.new
-            mv $i.new $i
-          done
-          # Copy client certificate because symlinks do not work on Windows.
-          cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem
-
-  "make files executable":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do
-            chmod +x $i
-          done
-
-  "install dependencies":
-    - command: shell.exec
-      params:
-        working_dir: "src"
-        script: |
-          ${PREPARE_SHELL}
-          file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh"
-          # Don't use ${file} syntax here because evergreen treats it as an empty expansion.
-          [ -f "$file" ] && bash $file || echo "$file not available, skipping"
-
-pre:
-  - func: "fetch source"
-  - func: "prepare resources"
-  # We don't run perf on Windows (yet)
-  #- func: "windows fix"
-  - func: "fix absolute paths"
-  - func: "make files executable"
-  # We're not testing with TLS (yet)
-  #- func: "install dependencies"
-
-post:
-  - func: "stop mongo-orchestration"
-  - func: "cleanup"
-
-tasks:
-    - name: "perf-6.0-standalone"
-      tags: ["perf"]
-      commands:
-        - func: "bootstrap mongo-orchestration"
-          vars:
-            VERSION: "v6.0-perf"
-            TOPOLOGY: "server"
-        - func: "run perf tests"
-        - func: "attach benchmark test results"
-        - func: "send dashboard data"
-
-buildvariants:
-
- name: "perf-tests"
-  display_name: "Performance Benchmark Tests"
-  batchtime: 10080  # 7 days
-  run_on: rhel90-dbx-perf-large
-  tasks:
-     - name: "perf-6.0-standalone"
--- a/.evergreen/run-perf-tests.sh
+++ b/.evergreen/run-perf-tests.sh
@ -3,14 +3,14 @@
 set -o xtrace
 set -o errexit

-git clone https://github.com/mongodb-labs/driver-performance-test-data.git
-cd driver-performance-test-data
+git clone --depth 1 https://github.com/mongodb/specifications.git
+pushd specifications/source/benchmarking/data
 tar xf extended_bson.tgz
 tar xf parallel.tgz
 tar xf single_and_multi_document.tgz
-cd ..
+popd

-export TEST_PATH="${PROJECT_DIRECTORY}/driver-performance-test-data"
+export TEST_PATH="${PROJECT_DIRECTORY}/specifications/source/benchmarking/data"
 export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"

 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
--- a/test/performance/perf_test.py
+++ b/test/performance/perf_test.py
@ -83,9 +83,10 @@ class PerformanceTest:
        pass

    def tearDown(self):
-        name = self.__class__.__name__
+        # Remove "Test" so that TestFlatEncoding is reported as "FlatEncoding".
+        name = self.__class__.__name__[4:]
        median = self.percentile(50)
-        bytes_per_sec = self.data_size / median
+        megabytes_per_sec = self.data_size / median / 1000000
        print(f"Running {self.__class__.__name__}. MEDIAN={self.percentile(50)}")
        result_data.append(
            {
@ -96,7 +97,7 @@ class PerformanceTest:
                    },
                },
                "metrics": [
-                    {"name": "bytes_per_sec", "value": bytes_per_sec},
+                    {"name": "megabytes_per_sec", "type": "MEDIAN", "value": megabytes_per_sec},
                ],
            }
        )