diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index a28cd2596..7d9ab2df3 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -162,9 +162,45 @@ functions: # Send dashboard data send dashboard data: - - command: perf.send + - command: subprocess.exec params: - file: src/results.json + binary: bash + args: + - .evergreen/scripts/perf-submission-setup.sh + working_dir: src + include_expansions_in_env: + - requester + - revision_order_id + - project_id + - version_id + - build_variant + - parsed_order_id + - task_name + - task_id + - execution + - is_mainline + type: test + - command: expansions.update + params: + file: src/expansion.yml + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/perf-submission.sh + working_dir: src + include_expansions_in_env: + - requester + - revision_order_id + - project_id + - version_id + - build_variant + - parsed_order_id + - task_name + - task_id + - execution + - is_mainline + type: test # Setup system setup system: diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 5083b456c..eb5b49e6f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -134,7 +134,7 @@ tasks: commands: - func: download and merge coverage depends_on: [{ name: .server-version, variant: .coverage_tag, status: "*", patch_optional: true }] - tags: [coverage] + tags: [coverage, pr] # Free threading tests - name: test-free-threading @@ -211,7 +211,7 @@ tasks: TEST_NAME: mod_wsgi SUB_TEST_NAME: standalone PYTHON_VERSION: "3.9" - tags: [mod_wsgi] + tags: [mod_wsgi, pr] - name: mod-wsgi-embedded-mode-replica-set-python3.10 commands: - func: run server @@ -223,7 +223,7 @@ tasks: TEST_NAME: mod_wsgi SUB_TEST_NAME: embedded PYTHON_VERSION: "3.10" - tags: [mod_wsgi] + tags: [mod_wsgi, pr] - name: mod-wsgi-replica-set-python3.11 commands: - func: run server @@ -235,7 +235,7 @@ tasks: TEST_NAME: mod_wsgi SUB_TEST_NAME: standalone PYTHON_VERSION: "3.11" - tags: [mod_wsgi] + tags: [mod_wsgi, pr] - name: mod-wsgi-embedded-mode-replica-set-python3.12 commands: - func: run server @@ -247,7 +247,7 @@ tasks: TEST_NAME: mod_wsgi SUB_TEST_NAME: embedded PYTHON_VERSION: "3.12" - tags: [mod_wsgi] + tags: [mod_wsgi, pr] - name: mod-wsgi-replica-set-python3.13 commands: - func: run server @@ -259,7 +259,7 @@ tasks: TEST_NAME: mod_wsgi SUB_TEST_NAME: standalone PYTHON_VERSION: "3.13" - tags: [mod_wsgi] + tags: [mod_wsgi, pr] # No orchestration tests - name: test-no-orchestration-python3.9 @@ -2490,6 +2490,7 @@ tasks: - python-3.9 - standalone-noauth-nossl - sync + - pr - name: test-server-version-python3.10-async-noauth-nossl-standalone-cov commands: - func: run server @@ -2511,6 +2512,7 @@ tasks: - python-3.10 - standalone-noauth-nossl - async + - pr - name: test-server-version-python3.11-sync-auth-ssl-replica-set-cov commands: - func: run server @@ -2656,6 +2658,7 @@ tasks: - python-3.11 - replica_set-noauth-nossl - sync + - pr - name: test-server-version-python3.12-async-noauth-nossl-replica-set-cov commands: - func: run server @@ -2677,6 +2680,7 @@ tasks: - python-3.12 - replica_set-noauth-nossl - async + - pr - name: test-server-version-python3.13-sync-auth-ssl-sharded-cluster-cov commands: - func: run server @@ -2698,6 +2702,7 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - sync + - pr - name: test-server-version-pypy3.10-async-auth-ssl-sharded-cluster commands: - func: run server @@ -2717,6 +2722,7 @@ tasks: - python-pypy3.10 - sharded_cluster-auth-ssl - async + - pr - name: test-server-version-python3.9-sync-auth-nossl-sharded-cluster-cov commands: - func: run server @@ -3611,6 +3617,7 @@ tasks: - python-3.13 - standalone-noauth-nossl - sync + - pr - name: test-standard-latest-python3.9-async-noauth-ssl-replica-set commands: - func: run server @@ -3633,6 +3640,7 @@ tasks: - python-3.9 - replica_set-noauth-ssl - async + - pr - name: test-standard-latest-python3.10-sync-auth-ssl-sharded-cluster commands: - func: run server @@ -3655,6 +3663,7 @@ tasks: - python-3.10 - sharded_cluster-auth-ssl - sync + - pr - name: test-standard-v4.0-pypy3.10-sync-noauth-nossl-standalone commands: - func: run server @@ -4389,6 +4398,7 @@ tasks: - python-3.13 - standalone-noauth-nossl - noauth + - pr - name: test-non-standard-latest-python3.9-noauth-ssl-replica-set commands: - func: run server @@ -4410,6 +4420,7 @@ tasks: - python-3.9 - replica_set-noauth-ssl - noauth + - pr - name: test-non-standard-latest-python3.10-auth-ssl-sharded-cluster commands: - func: run server @@ -4431,6 +4442,7 @@ tasks: - python-3.10 - sharded_cluster-auth-ssl - auth + - pr - name: test-non-standard-v4.0-pypy3.10-noauth-nossl-standalone commands: - func: run server diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index e13976d8c..8bb4d5b21 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -41,7 +41,6 @@ from shrub.v3.evg_command import ( ec2_assume_role, expansions_update, git_get_project, - perf_send, ) from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef @@ -553,8 +552,19 @@ def create_server_version_tasks(): task_inputs.append(task_input) # Assemble the tasks. + seen = set() for topology, auth, ssl, sync, python in task_inputs: - tags = ["server-version", f"python-{python}", f"{topology}-{auth}-{ssl}", sync] + combo = f"{topology}-{auth}-{ssl}" + tags = ["server-version", f"python-{python}", combo, sync] + if combo in [ + "standalone-noauth-nossl", + "replica_set-noauth-nossl", + "sharded_cluster-auth-ssl", + ]: + combo = f"{combo}-{sync}" + if combo not in seen: + seen.add(combo) + tags.append("pr") expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology) if python not in PYPYS: expansions["COVERAGE"] = "1" @@ -593,11 +603,12 @@ def create_test_non_standard_tasks(): task_combos = [] # For each version and topology, rotate through the CPythons. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): - task_combos.append((version, topology, python)) + pr = version == "latest" + task_combos.append((version, topology, python, pr)) # For each PyPy and topology, rotate through the the versions. for (python, topology), version in zip_cycle(list(product(PYPYS, TOPOLOGIES)), ALL_VERSIONS): - task_combos.append((version, topology, python)) - for version, topology, python in task_combos: + task_combos.append((version, topology, python, False)) + for version, topology, python, pr in task_combos: auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-non-standard", @@ -608,6 +619,8 @@ def create_test_non_standard_tasks(): ] if python in PYPYS: tags.append("pypy") + if pr: + tags.append("pr") expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -626,14 +639,15 @@ def create_standard_tasks(): for (version, topology), python, sync in zip_cycle( list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS, SYNCS ): - task_combos.append((version, topology, python, sync)) + pr = version == "latest" + task_combos.append((version, topology, python, sync, pr)) # For each PyPy and topology, rotate through the the versions and sync/async. for (python, topology), version, sync in zip_cycle( list(product(PYPYS, TOPOLOGIES)), ALL_VERSIONS, SYNCS ): - task_combos.append((version, topology, python, sync)) + task_combos.append((version, topology, python, sync, False)) - for version, topology, python, sync in task_combos: + for version, topology, python, sync, pr in task_combos: auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard", @@ -644,6 +658,8 @@ def create_standard_tasks(): ] if python in PYPYS: tags.append("pypy") + if pr: + tags.append("pr") expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -758,7 +774,7 @@ def create_mod_wsgi_tasks(): server_func = FunctionCall(func="run server", vars=server_vars) vars = dict(TEST_NAME="mod_wsgi", SUB_TEST_NAME=test.split("-")[0], PYTHON_VERSION=python) test_func = FunctionCall(func="run tests", vars=vars) - tags = ["mod_wsgi"] + tags = ["mod_wsgi", "pr"] commands = [server_func, test_func] tasks.append(EvgTask(name=task_name, tags=tags, commands=commands)) return tasks @@ -843,7 +859,7 @@ def create_getdata_tasks(): def create_coverage_report_tasks(): - tags = ["coverage"] + tags = ["coverage", "pr"] task_name = "coverage-report" # BUILD-3165: We can't use "*" (all tasks) and specify "variant". # Instead list out all coverage tasks using tags. @@ -1103,8 +1119,28 @@ def create_attach_benchmark_test_results_func(): def create_send_dashboard_data_func(): - cmd = perf_send(file="src/results.json") - return "send dashboard data", [cmd] + includes = [ + "requester", + "revision_order_id", + "project_id", + "version_id", + "build_variant", + "parsed_order_id", + "task_name", + "task_id", + "execution", + "is_mainline", + ] + cmds = [ + get_subprocess_exec( + include_expansions_in_env=includes, args=[".evergreen/scripts/perf-submission-setup.sh"] + ), + expansions_update(file="src/expansion.yml"), + get_subprocess_exec( + include_expansions_in_env=includes, args=[".evergreen/scripts/perf-submission.sh"] + ), + ] + return "send dashboard data", cmds mod = sys.modules[__name__] diff --git a/.evergreen/scripts/perf-submission-setup.sh b/.evergreen/scripts/perf-submission-setup.sh new file mode 100755 index 000000000..ecb38751a --- /dev/null +++ b/.evergreen/scripts/perf-submission-setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# We use the requester expansion to determine whether the data is from a mainline evergreen run or not + +set -eu + +# shellcheck disable=SC2154 +if [ "${requester}" == "commit" ]; then + echo "is_mainline: true" >> expansion.yml +else + echo "is_mainline: false" >> expansion.yml +fi + +# We parse the username out of the order_id as patches append that in and SPS does not need that information +# shellcheck disable=SC2154 +echo "parsed_order_id: $(echo "${revision_order_id}" | awk -F'_' '{print $NF}')" >> expansion.yml diff --git a/.evergreen/scripts/perf-submission.sh b/.evergreen/scripts/perf-submission.sh new file mode 100755 index 000000000..f7c3ea666 --- /dev/null +++ b/.evergreen/scripts/perf-submission.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# We use the requester expansion to determine whether the data is from a mainline evergreen run or not + +set -eu + +# Submit the performance data to the SPS endpoint +# shellcheck disable=SC2154 +response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \ + "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=${parsed_order_id}&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=${is_mainline}" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d @results.json) + +http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}') +response_body=$(echo "$response" | sed '/HTTP_STATUS/d') + +# We want to throw an error if the data was not successfully submitted +if [ "$http_status" -ne 200 ]; then + echo "Error: Received HTTP status $http_status" + echo "Response Body: $response_body" + exit 1 +fi + +echo "Response Body: $response_body" +echo "HTTP Status: $http_status" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f7cfea144..3becfd72b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,7 +46,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3 + uses: github/codeql-action/init@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -63,6 +63,6 @@ jobs: pip install -e . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3 + uses: github/codeql-action/analyze@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 05b87f4f8..f6056f6ca 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -25,7 +25,7 @@ jobs: - name: Install just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: "3.9" @@ -65,7 +65,7 @@ jobs: - name: Install just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: ${{ matrix.python-version }} @@ -88,7 +88,7 @@ jobs: - name: Install just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: "3.9" @@ -111,7 +111,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: "3.9" @@ -130,7 +130,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: "3.9" @@ -152,7 +152,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v5 with: enable-cache: true python-version: "${{matrix.python}}" diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 16f2ba2cb..b0d4e7cf2 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,7 +26,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3 + uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3 with: sarif_file: results.sarif category: zizmor diff --git a/test/performance/async_perf_test.py b/test/performance/async_perf_test.py index 969437f9c..6eb31ea4f 100644 --- a/test/performance/async_perf_test.py +++ b/test/performance/async_perf_test.py @@ -144,7 +144,15 @@ class PerformanceTest: }, }, "metrics": [ - {"name": "megabytes_per_sec", "type": "MEDIAN", "value": megabytes_per_sec}, + { + "name": "megabytes_per_sec", + "type": "MEDIAN", + "value": megabytes_per_sec, + "metadata": { + "improvement_direction": "up", + "measurement_unit": "megabytes_per_second", + }, + }, ], } ) diff --git a/test/performance/perf_test.py b/test/performance/perf_test.py index 39487eff6..5688d28d2 100644 --- a/test/performance/perf_test.py +++ b/test/performance/perf_test.py @@ -151,7 +151,15 @@ class PerformanceTest: }, }, "metrics": [ - {"name": "megabytes_per_sec", "type": "MEDIAN", "value": megabytes_per_sec}, + { + "name": "megabytes_per_sec", + "type": "MEDIAN", + "value": megabytes_per_sec, + "metadata": { + "improvement_direction": "up", + "measurement_unit": "megabytes_per_second", + }, + }, ], } )