From cc83d593a14bb924718de7827d9cded82ecf80eb Mon Sep 17 00:00:00 2001 From: Finley Lau Date: Wed, 11 Feb 2026 16:35:22 -0500 Subject: [PATCH] SERVER-118486 Update api_boundary_server_status.js to use extension metric helpers (#47942) GitOrigin-RevId: daa39f7f731e44421bbfb71c08463a762a01ea7f --- .../extensions/api_boundary_server_status.js | 346 +++++++++--------- .../extension_stage_command_metrics.js | 53 ++- jstests/extensions/libs/BUILD.bazel | 12 + .../libs/extension_metrics_helpers.js | 121 ++++++ 4 files changed, 334 insertions(+), 198 deletions(-) create mode 100644 jstests/extensions/libs/BUILD.bazel create mode 100644 jstests/extensions/libs/extension_metrics_helpers.js diff --git a/jstests/extensions/api_boundary_server_status.js b/jstests/extensions/api_boundary_server_status.js index 37739fb7f25..7a9c0e8da4a 100644 --- a/jstests/extensions/api_boundary_server_status.js +++ b/jstests/extensions/api_boundary_server_status.js @@ -12,6 +12,7 @@ */ import {FixtureHelpers} from "jstests/libs/fixture_helpers.js"; import {after, before, describe, it} from "jstests/libs/mochalite.js"; +import {observeExtensionMetricsChange} from "jstests/extensions/libs/extension_metrics_helpers.js"; // The 'serverStatus' command is unreliable in test suites with multiple mongos processes given that // each node has its own metrics. The assertions here would not hold up if run against multiple @@ -19,7 +20,7 @@ import {after, before, describe, it} from "jstests/libs/mochalite.js"; TestData.pinToSingleMongos = true; /** - * Helper to get an extension server status metric. + * Helper to get an extension server status metric from a specific node. */ function getExtensionMetricFromNode(conn, metric) { const serverStatus = conn.getDB("admin").runCommand({serverStatus: 1}); @@ -43,28 +44,16 @@ function getTotalExtensionMetrics(testDb, metric) { return total; } -function getTotalExtensionSuccesses(testDb) { - return getTotalExtensionMetrics(testDb, "extensionSuccesses"); -} - -function getTotalExtensionFailures(testDb) { - return getTotalExtensionMetrics(testDb, "extensionFailures"); -} - -function getTotalHostSuccesses(testDb) { - return getTotalExtensionMetrics(testDb, "hostSuccesses"); -} - -function getTotalHostFailures(testDb) { - return getTotalExtensionMetrics(testDb, "hostFailures"); -} - +/** + * Gets all relevant extension metrics aggregated across all nodes. + * Returns an object with extensionSuccesses, extensionFailures, hostSuccesses, and hostFailures. + */ function getTotalMetrics(testDB) { return { - "extensionSuccesses": getTotalExtensionSuccesses(testDB), - "extensionFailures": getTotalExtensionFailures(testDB), - "hostSuccesses": getTotalHostSuccesses(testDB), - "hostFailures": getTotalHostFailures(testDB), + extensionSuccesses: getTotalExtensionMetrics(testDB, "extensionSuccesses"), + extensionFailures: getTotalExtensionMetrics(testDB, "extensionFailures"), + hostSuccesses: getTotalExtensionMetrics(testDB, "hostSuccesses"), + hostFailures: getTotalExtensionMetrics(testDB, "hostFailures"), }; } @@ -84,251 +73,270 @@ describe("Extension success and failure serverStatus metrics", function () { after(function () { this.coll.drop(); }); - // TODO SERVER-118486: Leverage observeExtensionMetricsChange() helper in this jstest, modifying the helper as needed. + it("should have non-negative initial values", function () { const initialMetrics = getTotalMetrics(db); - assert.gte(initialMetrics["extensionSuccesses"], 0, `extensionSuccesses should be non-negative.`); - assert.gte(initialMetrics["extensionFailures"], 0, `extensionFailures should be non-negative.`); - assert.gte(initialMetrics["hostSuccesses"], 0, `hostSuccesses should be non-negative.`); - assert.gte(initialMetrics["hostFailures"], 0, `hostFailures should be non-negative.`); + assert.gte(initialMetrics.extensionSuccesses, 0, `extensionSuccesses should be non-negative.`); + assert.gte(initialMetrics.extensionFailures, 0, `extensionFailures should be non-negative.`); + assert.gte(initialMetrics.hostSuccesses, 0, `hostSuccesses should be non-negative.`); + assert.gte(initialMetrics.hostFailures, 0, `hostFailures should be non-negative.`); }); it("should increase extensionSuccesses when $testFoo runs successfully", function () { - const beforeMetrics = getTotalMetrics(db); + const result = observeExtensionMetricsChange( + [ + { + operation: () => this.coll.aggregate([{$testFoo: {}}]).toArray(), + expectSuccess: true, + }, + ], + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - // Run a query with $testFoo early in pipeline (runs on shards in sharded env). - const result = this.coll.aggregate([{$testFoo: {}}]).toArray(); - assert.eq(result.length, this.numDocs, "Query should return all documents"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nSuccessfulOperations, 1, "Query should succeed"); assert.gt( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, "extensionSuccesses should increase after successful extension work.", ); assert.eq( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should remain stable after successful extension work.`, ); // Note, hostSuccesses increases anytime the extension calls back into the host. This can be // an indeterminate number of times in the successful case. assert.gte( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], - `hostSuccesses should increase after successful host work.`, - ); - assert.eq( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], - `hostFailures should remain stable after successful host work.`, + result.metricDeltas.hostSuccesses, + 0, + `hostSuccesses should not decrease after successful host work.`, ); + assert.eq(result.metricDeltas.hostFailures, 0, `hostFailures should remain stable after successful host work.`); }); it("should increase extensionSuccesses when $testFoo runs on mongos (after $sort)", function () { - const beforeMetrics = getTotalMetrics(db); + const result = observeExtensionMetricsChange( + [ + { + operation: () => this.coll.aggregate([{$sort: {value: 1}}, {$testFoo: {}}]).toArray(), + expectSuccess: true, + }, + ], + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - // Run a query with $testFoo after $sort. - // In sharded clusters, $sort causes merging on mongos, so $testFoo runs on mongos. - const result = this.coll.aggregate([{$sort: {value: 1}}, {$testFoo: {}}]).toArray(); - assert.eq(result.length, this.numDocs, "Query should return all documents"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nSuccessfulOperations, 1, "Query should succeed"); assert.gt( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, "extensionSuccesses should increase after successful extension work.", ); assert.eq( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should remain stable after successful extension work.`, ); - assert.gt( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], - `hostSuccesses should increase after successful host work.`, - ); - assert.eq( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], - `hostFailures should remain stable after successful host work.`, - ); + assert.gt(result.metricDeltas.hostSuccesses, 0, `hostSuccesses should increase after successful host work.`); + assert.eq(result.metricDeltas.hostFailures, 0, `hostFailures should remain stable after successful host work.`); }); it("should accumulate extensionSuccesses across multiple queries", function () { const numQueries = 3; + const operations = []; for (let i = 0; i < numQueries; i++) { - const beforeMetrics = getTotalMetrics(db); - - const result = this.coll.aggregate([{$testFoo: {}}]).toArray(); - assert.eq(result.length, this.numDocs, `Query ${i + 1} should return all documents`); - const afterMetrics = getTotalMetrics(db); - assert.gt( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], - "extensionSuccesses should accumulate across multiple queries.", - ); - assert.eq( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], - `extensionFailures should remain stable across multiple successful queries`, - ); - // Note, hostSuccesses increases anytime the extension calls back into the host. This - // can be an indeterminate number of times in the succesful case. - assert.gte( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], - `hostSuccesses should increase across multiple queries.`, - ); - assert.eq( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], - `hostFailures should remain stable across multiple successful queries.`, - ); + operations.push({ + operation: () => this.coll.aggregate([{$testFoo: {}}]).toArray(), + expectSuccess: true, + }); } + + const result = observeExtensionMetricsChange(operations, () => getTotalMetrics(db), [ + "extensionSuccesses", + "extensionFailures", + "hostSuccesses", + "hostFailures", + ]); + + assert.eq(result.nSuccessfulOperations, numQueries, "All queries should succeed"); + assert.gt( + result.metricDeltas.extensionSuccesses, + 0, + "extensionSuccesses should accumulate across multiple queries.", + ); + assert.eq( + result.metricDeltas.extensionFailures, + 0, + `extensionFailures should remain stable across multiple successful queries`, + ); + // Note, hostSuccesses increases anytime the extension calls back into the host. This + // can be an indeterminate number of times in the successful case. + assert.gte(result.metricDeltas.hostSuccesses, 0, `hostSuccesses should not decrease across multiple queries.`); + assert.eq( + result.metricDeltas.hostFailures, + 0, + `hostFailures should remain stable across multiple successful queries.`, + ); }); it("extension and host failures should both increase when $assert triggers a uassert in parse phase", function () { - const beforeMetrics = getTotalMetrics(db); - - // Run a query with $assert that will trigger a uassert failure at parse time. - const assertResult = db.runCommand({ - aggregate: this.coll.getName(), - pipeline: [ + const result = observeExtensionMetricsChange( + [ { - $assert: {errmsg: "test uassert failure", code: 11569609, assertionType: "uassert"}, + operation: () => + db.runCommand({ + aggregate: this.coll.getName(), + pipeline: [ + { + $assert: {errmsg: "test uassert failure", code: 11569609, assertionType: "uassert"}, + }, + ], + cursor: {}, + }), + expectSuccess: false, }, ], - cursor: {}, - }); + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - assert.commandFailedWithCode(assertResult, 11569609, "Assert should fail with expected code"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nFailedOperations, 1, "Operation should fail"); assert.gte( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, `extensionSuccesses should not decrease after extension failure at parse time.`, ); assert.gt( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should increase after extension failure at parse time.`, ); assert.gte( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], + result.metricDeltas.hostSuccesses, + 0, `hostSuccesses should not decrease after extension failure at parse time.`, ); assert.gt( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], + result.metricDeltas.hostFailures, + 0, `hostFailures should increase due to host triggered uassert at parse time.`, ); }); it("should increase both extensionSuccess and extensionFailures when $assert triggers a uassert in ast phase", function () { - const beforeMetrics = getTotalMetrics(db); - - // Run a query with $assert that will trigger a uassert failure at ast creation time. - const assertResult = db.runCommand({ - aggregate: this.coll.getName(), - pipeline: [ + const result = observeExtensionMetricsChange( + [ { - $assert: { - errmsg: "test uassert failure in ast phase", - code: 11569610, - assertionType: "uassert", - assertInPhase: "ast", - }, + operation: () => + db.runCommand({ + aggregate: this.coll.getName(), + pipeline: [ + { + $assert: { + errmsg: "test uassert failure in ast phase", + code: 11569610, + assertionType: "uassert", + assertInPhase: "ast", + }, + }, + ], + cursor: {}, + }), + expectSuccess: false, }, ], - cursor: {}, - }); + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - assert.commandFailedWithCode(assertResult, 11569610, "Assert should fail with expected code"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nFailedOperations, 1, "Operation should fail"); assert.gt( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, `extensionSuccesses should increase after extension failure in ast phase.`, ); assert.gt( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should increase after extension failure in ast phase.`, ); assert.gte( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], + result.metricDeltas.hostSuccesses, + 0, `hostSuccesses should not decrease after extension failure in ast phase.`, ); assert.gt( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], + result.metricDeltas.hostFailures, + 0, `hostFailures should increase due to host triggered uassert in ast phase.`, ); }); it("should NOT increase extensionFailures for successful queries", function () { - const beforeMetrics = getTotalMetrics(db); - // extension failures should not increase + const result = observeExtensionMetricsChange( + [ + { + operation: () => this.coll.aggregate([{$testFoo: {}}]).toArray(), + expectSuccess: true, + }, + ], + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - const result = this.coll.aggregate([{$testFoo: {}}]).toArray(); - assert.eq(result.length, this.numDocs, "Query should return all documents"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nSuccessfulOperations, 1, "Query should succeed"); assert.gt( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, `extensionSuccesses should increase after successful extension work.`, ); assert.eq( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should remain stable after successful extension work.`, ); // Note, hostSuccesses increases anytime the extension calls back into the host. This can be - // an indeterminate number of times in the succesful case. + // an indeterminate number of times in the successful case. assert.gte( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], - `hostSuccesses should increase after successful host work.`, - ); - assert.eq( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], - `hostFailures should remain stable after successful host work.`, + result.metricDeltas.hostSuccesses, + 0, + `hostSuccesses should not decrease after successful host work.`, ); + assert.eq(result.metricDeltas.hostFailures, 0, `hostFailures should remain stable after successful host work.`); }); it("should NOT increase extensionSuccesses for queries without extension stages", function () { - const beforeMetrics = getTotalMetrics(db); + const result = observeExtensionMetricsChange( + [ + { + operation: () => this.coll.aggregate([{$match: {value: {$gte: 0}}}]).toArray(), + expectSuccess: true, + }, + ], + () => getTotalMetrics(db), + ["extensionSuccesses", "extensionFailures", "hostSuccesses", "hostFailures"], + ); - const result = this.coll.aggregate([{$match: {value: {$gte: 0}}}]).toArray(); - assert.eq(result.length, this.numDocs, "Query should return all documents"); - - const afterMetrics = getTotalMetrics(db); + assert.eq(result.nSuccessfulOperations, 1, "Query should succeed"); assert.eq( - afterMetrics["extensionSuccesses"], - beforeMetrics["extensionSuccesses"], + result.metricDeltas.extensionSuccesses, + 0, `extensionSuccesses should remain stable for query not using extension stages.`, ); assert.eq( - afterMetrics["extensionFailures"], - beforeMetrics["extensionFailures"], + result.metricDeltas.extensionFailures, + 0, `extensionFailures should remain stable for query not using extension stages.`, ); assert.eq( - afterMetrics["hostSuccesses"], - beforeMetrics["hostSuccesses"], + result.metricDeltas.hostSuccesses, + 0, `hostSuccesses should remain stable for query not using extension stages.`, ); assert.eq( - afterMetrics["hostFailures"], - beforeMetrics["hostFailures"], + result.metricDeltas.hostFailures, + 0, `hostFailures should remain stable for query not using extension stages.`, ); }); diff --git a/jstests/extensions/extension_stage_command_metrics.js b/jstests/extensions/extension_stage_command_metrics.js index 84f8cad0cd3..6dc5f753dbd 100644 --- a/jstests/extensions/extension_stage_command_metrics.js +++ b/jstests/extensions/extension_stage_command_metrics.js @@ -26,6 +26,7 @@ */ import {after, before, beforeEach, describe, it} from "jstests/libs/mochalite.js"; import {assertDropCollection} from "jstests/libs/collection_drop_recreate.js"; +import {observeExtensionMetricsChange} from "jstests/extensions/libs/extension_metrics_helpers.js"; const collName = jsTestName(); @@ -47,45 +48,39 @@ function getExtensionCommandMetrics() { return serverStatus.metrics.commands.aggregate.withExtension; } -function observeExtensionMetricsChange(pipelinesToRun) { - const metricsBefore = getExtensionCommandMetrics(); - let nSuccesses = 0, - nFailures = 0; - for (const {coll, pipeline} of pipelinesToRun) { - try { - coll.aggregate(pipeline); - nSuccesses++; - } catch (e) { - nFailures++; - } - } - const metricsAfter = getExtensionCommandMetrics(); - return { - nSuccessfulPipelines: nSuccesses, - successMetricDelta: metricsAfter.succeeded - metricsBefore.succeeded, - nFailedPipelines: nFailures, - failureMetricDelta: metricsAfter.failed - metricsBefore.failed, - }; -} - /** * Verifies that the extension command metrics are changed by the provided pipelines. */ function verifyExtensionMetricsChange(pipelinesToRun) { - const {nSuccessfulPipelines, successMetricDelta, nFailedPipelines, failureMetricDelta} = - observeExtensionMetricsChange(pipelinesToRun); - assert.eq(successMetricDelta, nSuccessfulPipelines); - assert.eq(failureMetricDelta, nFailedPipelines); + const operations = pipelinesToRun.map(({coll, pipeline}) => ({ + operation: () => coll.aggregate(pipeline), + })); + + const {nSuccessfulOperations, nFailedOperations, metricDeltas} = observeExtensionMetricsChange( + operations, + getExtensionCommandMetrics, + ["succeeded", "failed"], + ); + + assert.eq(metricDeltas.succeeded, nSuccessfulOperations); + assert.eq(metricDeltas.failed, nFailedOperations); } /** * Verifies that the extension command metrics are *not* changed by the provided pipelines. */ function verifyExtensionMetricsDoNotChange(pipelinesToRun) { - const {nSuccessfulPipelines, successMetricDelta, nFailedPipelines, failureMetricDelta} = - observeExtensionMetricsChange(pipelinesToRun); - assert.eq(successMetricDelta, 0); - assert.eq(failureMetricDelta, 0); + const operations = pipelinesToRun.map(({coll, pipeline}) => ({ + operation: () => coll.aggregate(pipeline), + })); + + const {metricDeltas} = observeExtensionMetricsChange(operations, getExtensionCommandMetrics, [ + "succeeded", + "failed", + ]); + + assert.eq(metricDeltas.succeeded, 0); + assert.eq(metricDeltas.failed, 0); } describe("Extension stage command metrics", function () { diff --git a/jstests/extensions/libs/BUILD.bazel b/jstests/extensions/libs/BUILD.bazel new file mode 100644 index 00000000000..97513de081b --- /dev/null +++ b/jstests/extensions/libs/BUILD.bazel @@ -0,0 +1,12 @@ +load("//bazel:mongo_js_rules.bzl", "all_subpackage_javascript_files", "mongo_js_library") + +package(default_visibility = ["//visibility:public"]) + +mongo_js_library( + name = "all_javascript_files", + srcs = glob([ + "*.js", + ]), +) + +all_subpackage_javascript_files() diff --git a/jstests/extensions/libs/extension_metrics_helpers.js b/jstests/extensions/libs/extension_metrics_helpers.js new file mode 100644 index 00000000000..3267384af04 --- /dev/null +++ b/jstests/extensions/libs/extension_metrics_helpers.js @@ -0,0 +1,121 @@ +/** + * Helper utilities for tracking and observing extension metrics in tests. + * + * These utilities support multiple metric sources (e.g., command-level metrics, extension-level + * metrics) and can aggregate metrics across multiple nodes in sharded or replica set environments. + */ + +/** + * Observes changes in extension metrics while running the provided operations. + * + * This is a flexible helper that can track any numeric metrics by comparing their values + * before and after running operations. It supports both successful and failing operations. + * + * @param {Array} operationsToRun - Array of operations to execute. Each operation should have: + * - operation: A function that performs the operation and returns a result + * - expectSuccess: (Optional) Boolean indicating whether the operation is expected to succeed. + * If provided, the helper will assert that the actual outcome matches. + * If not provided (null/undefined), no expectation validation is performed. + * + * @param {Function} getMetricsFn - Function that retrieves the current metrics object. + * The object should have numeric properties that can be compared before/after. + * This function is called twice: once before running operations and once after. + * + * @param {Array} metricsToTrack - Array of metric property names to track. + * These should correspond to numeric properties in the object returned by getMetricsFn. + * Example: ['succeeded', 'failed'] or ['extensionSuccesses', 'extensionFailures'] + * + * @returns {Object} An object containing: + * - nSuccessfulOperations: Number of operations that succeeded (did not throw) + * - nFailedOperations: Number of operations that failed (threw an exception) + * - metricDeltas: Object mapping each tracked metric name to its delta (after - before) + * + * @example + * // Track command-level metrics from a single node with expectation validation + * function getExtensionCommandMetrics() { + * const serverStatus = assert.commandWorked(db.adminCommand({serverStatus: 1})); + * return serverStatus.metrics.commands.aggregate.withExtension; + * } + * + * const result = observeExtensionMetricsChange( + * [{operation: () => coll.aggregate([{$metrics: {}}]), expectSuccess: true}], + * getExtensionCommandMetrics, + * ['succeeded', 'failed'] + * ); + * + * assert.eq(result.metricDeltas.succeeded, 1); + * assert.eq(result.metricDeltas.failed, 0); + * + * @example + * // Track extension-level metrics without expectation validation + * function getTotalMetrics(testDB) { + * return { + * extensionSuccesses: getTotalExtensionMetrics(testDB, "extensionSuccesses"), + * extensionFailures: getTotalExtensionMetrics(testDB, "extensionFailures"), + * }; + * } + * + * const result = observeExtensionMetricsChange( + * [{operation: () => coll.aggregate([{$testFoo: {}}]).toArray()}], // no expectSuccess + * () => getTotalMetrics(db), + * ['extensionSuccesses', 'extensionFailures'] + * ); + * + * assert.gt(result.metricDeltas.extensionSuccesses, 0); + * + * @example + * // Track multiple operations with mixed success/failure expectations + * const result = observeExtensionMetricsChange( + * [ + * {operation: () => coll.aggregate([{$metrics: {}}]), expectSuccess: true}, + * {operation: () => coll.aggregate([{$invalidStage: {}}]), expectSuccess: false}, + * ], + * getExtensionCommandMetrics, + * ['succeeded', 'failed'] + * ); + * + * assert.eq(result.nSuccessfulOperations, 1); + * assert.eq(result.nFailedOperations, 1); + */ +export function observeExtensionMetricsChange(operationsToRun, getMetricsFn, metricsToTrack) { + const metricsBefore = getMetricsFn(); + let nSuccesses = 0, + nFailures = 0; + + for (let i = 0; i < operationsToRun.length; i++) { + const {operation, expectSuccess} = operationsToRun[i]; + const expectSuccessExplicitlySet = expectSuccess !== null && expectSuccess !== undefined; + try { + operation(); + nSuccesses++; + + // If expectSuccess is explicitly set to false, the operation should have failed. + if (expectSuccessExplicitlySet && expectSuccess === false) { + throw new Error( + `Operation ${i} was expected to fail but succeeded. ` + + `Set expectSuccess to true or omit it if success is acceptable.`, + ); + } + } catch (e) { + nFailures++; + + // If expectSuccess is explicitly set to true, the operation should have succeeded. + if (expectSuccessExplicitlySet && expectSuccess === true) { + throw new Error(`Operation ${i} was expected to succeed but failed with error: ${e.message}`); + } + } + } + + const metricsAfter = getMetricsFn(); + + const metricDeltas = {}; + for (const metricName of metricsToTrack) { + metricDeltas[metricName] = metricsAfter[metricName] - metricsBefore[metricName]; + } + + return { + nSuccessfulOperations: nSuccesses, + nFailedOperations: nFailures, + metricDeltas: metricDeltas, + }; +}