From a126cda03abbbe918ef8aebfd0d1c5025710b4ed Mon Sep 17 00:00:00 2001 From: Gustavo Tenrreiro <123435846+mdb-gtenrreiro@users.noreply.github.com> Date: Wed, 4 Feb 2026 04:07:37 -0600 Subject: [PATCH] SERVER-115951: Adds External JS Server integration (#43960) GitOrigin-RevId: 241578dc158ad14b67a0dda1f4dda20d46d218ba --- .github/CODEOWNERS | 3 + .prettierignore | 2 + .../suites/streams_externaljs.yml | 31 +++++++ etc/evergreen_yml_components/definitions.yml | 86 +++++++++++++++++++ .../tasks/resmoke/non_server_teams/tasks.yml | 32 +++++-- .../amazon/streams/streams_release.yml | 2 + .../variants/amazon/test_release.yml | 1 - evergreen/streams_build_js_engine.sh | 7 +- src/mongo/scripting/BUILD.bazel | 6 ++ src/mongo/scripting/config.idl | 39 +++++++++ src/mongo/scripting/engine.h | 24 ++++-- src/mongo/scripting/engine_none.cpp | 4 - src/mongo/scripting/mozjs/shell/engine.cpp | 26 +++++- src/mongo/scripting/mozjs/shell/engine.h | 2 + 14 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 buildscripts/resmokeconfig/suites/streams_externaljs.yml create mode 100644 src/mongo/scripting/config.idl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 31797f0f034..f8d5c3b4669 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2386,6 +2386,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot # The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams/aspio/OWNERS.yml /src/mongo/db/modules/enterprise/jstests/streams/aspio/**/* @10gen/streams-engine @svc-auto-approve-bot +# The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams/externaljs/OWNERS.yml +/src/mongo/db/modules/enterprise/jstests/streams/externaljs/**/* @10gen/streams-engine @svc-auto-approve-bot + # The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams_kafka/OWNERS.yml /src/mongo/db/modules/enterprise/jstests/streams_kafka/**/* @10gen/streams-engine @svc-auto-approve-bot diff --git a/.prettierignore b/.prettierignore index 182ee258308..da35399feb0 100644 --- a/.prettierignore +++ b/.prettierignore @@ -64,3 +64,5 @@ bazel-* # Streams specific src/mongo/db/modules/enterprise/src/streams/third_party/mongocxx/dist +# asp-js-engine is an external module copied during Docker builds +asp-js-engine diff --git a/buildscripts/resmokeconfig/suites/streams_externaljs.yml b/buildscripts/resmokeconfig/suites/streams_externaljs.yml new file mode 100644 index 00000000000..931eb029788 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/streams_externaljs.yml @@ -0,0 +1,31 @@ +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/streams/externaljs/*.js + +executor: + config: + shell_options: + global_vars: + TestData: + # Prevent auto-execution of imported test modules. Tests in this suite + # use containerized mongostream and must control when tests run. + skipDefaultRun: true + fixture: + class: ReplicaSetFixture + mongod_options: + bind_ip_all: "" + set_parameters: + enableTestCommands: 1 + featureFlagStreams: true + diagnosticDataCollectionEnabled: false + # ExternalJS server parameters + # These can be overridden via --setParameter on the command line + enableExternalScripting: true + jsBinPath: "/usr/bin/node" + jsSvrPath: "/app/externaljs/dist/server.js" + jsSvrPort: 50051 + jsSvrWrkDir: "/tmp/" + jsSvrStartTimeoutMs: 10000 + num_nodes: 1 diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 6086bf0fc5f..9cb1ac854c3 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1254,6 +1254,23 @@ functions: OTEL_PARENT_ID: ${otel_parent_id} OTEL_COLLECTOR_DIR: "../build/OTelTraces/" + "execute resmoke tests with asp js engine": + &execute_resmoke_tests_with_asp_js_engine_token + command: subprocess.exec + display_name: "execute resmoke tests with asp js engine" + type: test + params: + binary: bash + args: + - "./src/evergreen/resmoke_tests_execute.sh" + env: + OTEL_TRACE_ID: ${otel_trace_id} + OTEL_PARENT_ID: ${otel_parent_id} + OTEL_COLLECTOR_DIR: "../build/OTelTraces/" + # Path to the asp-js-engine module cloned via Evergreen modules + ASP_JS_ENGINE_PATH: ${workdir}/asp-js-engine + HAS_JS_ENGINE: ${HAS_JS_ENGINE|} + "execute resmoke tests via bazel sh": &execute_resmoke_tests_via_bazel_sh command: subprocess.exec display_name: "execute resmoke tests via bazel sh" @@ -1658,6 +1675,75 @@ functions: - *check_run_tests_infrastructure_failure - *check_resmoke_failure + # Run streams tests that require access to the private asp-js-engine repo + # Clone the asp-js-engine module and restore git history/tags so resmoke's git describe works + # Re-fetch and extract binaries after git clone because the clone overwrites src/ (including tarballs and dist-test/) + # Copy asp-js-engine from src/ to workdir where resmoke expects it (ASP_JS_ENGINE_PATH=${workdir}/asp-js-engine) + "run streams tests": + - *f_expansions_write + - *git_get_shallow_streams_project + - *restore_git_history_and_tags + # Copy asp-js-engine module from src/ to workdir where ASP_JS_ENGINE_PATH expects it + - command: subprocess.exec + display_name: "copy asp-js-engine to workdir" + params: + binary: bash + args: + - "-c" + - | + set -o errexit + set -o verbose + # git.get_project clones into src/, so asp-js-engine module is at src/asp-js-engine/asp-js-engine + # ASP_JS_ENGINE_PATH expects it at ${workdir}/asp-js-engine with package.json at root + rm -rf asp-js-engine + cp -r src/asp-js-engine/asp-js-engine asp-js-engine + - *fetch_binaries + - *fetch_binaries_zstd + - *fetch_tgz_binary_shas + - *fetch_and_verify_binaries_sha + - *fetch_and_verify_binaries_sha_zstd + - *fetch_jstestshell + - *verify_jstestshell_sha + - *extract_binaries + - *extract_jstestshell + - *configure_evergreen_api_credentials + - *determine_task_timeout + - *update_task_timeout_expansions + - *f_expansions_write + - *update_task_timeout + - *f_expansions_write + - *set_code_coverage_expansion + - *f_expansions_write + - command: expansions.update + params: + env: + CEDAR_USER: ${cedar_user} + CEDAR_API_KEY: ${cedar_api_key} + updates: + - key: aws_key_remote + value: ${mongodatafiles_aws_key} + - key: aws_profile_remote + value: mongodata_aws + - key: aws_secret_remote + value: ${mongodatafiles_aws_secret} + - *f_expansions_write + - *set_up_remote_credentials + - *f_expansions_write + - *determine_resmoke_jobs + - *update_resmoke_jobs_expansions + - *f_expansions_write + - *configure_evergreen_api_credentials + - *sign_macos_dev_binaries + - *multiversion_exclude_tags_generate + - *assume_ecr_role + - *fetch_module_images + - *execute_resmoke_tests_with_asp_js_engine_token + # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't + # directly actionable. We use type=setup rather than type=system or type=test for this command + # because we don't intend for any human to look at this failure. + - *check_run_tests_infrastructure_failure + - *check_resmoke_failure + "run benchmark tests": - *f_expansions_write - *configure_evergreen_api_credentials diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index 4194304a2c6..4dd0f6c8b13 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -323,7 +323,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -338,7 +338,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -353,7 +353,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -368,7 +368,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -383,7 +383,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -398,7 +398,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 @@ -413,10 +413,26 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 + - <<: *task_template + name: streams_externaljs + tags: + [ + "assigned_to_jira_team_streams", + "default", + "streams_release_test", + "requires_extra_system_deps", + ] + commands: + - func: "do setup" + - func: "run streams tests" + vars: + resmoke_jobs_max: 1 + HAS_JS_ENGINE: "true" + - <<: *task_template name: streams_aspio_pubsub tags: @@ -428,7 +444,7 @@ tasks: ] commands: - func: "do setup" - - func: "run tests" + - func: "run streams tests" vars: resmoke_jobs_max: 1 diff --git a/etc/evergreen_yml_components/variants/amazon/streams/streams_release.yml b/etc/evergreen_yml_components/variants/amazon/streams/streams_release.yml index 9199cf8a0d0..044c75105b2 100644 --- a/etc/evergreen_yml_components/variants/amazon/streams/streams_release.yml +++ b/etc/evergreen_yml_components/variants/amazon/streams/streams_release.yml @@ -60,6 +60,7 @@ buildvariants: - name: streams_aspio_iceberg_3 - name: streams_aspio_iceberg_4 - name: streams_aspio_iceberg_5 + - name: streams_externaljs - name: streams_aspio_pubsub - name: streams_build_and_push_gen - name: streams_build_and_push_break_glass_gen @@ -118,6 +119,7 @@ buildvariants: - name: streams_aspio_iceberg_3 - name: streams_aspio_iceberg_4 - name: streams_aspio_iceberg_5 + - name: streams_externaljs - name: streams_aspio_pubsub - name: streams_build_and_push_gen - name: streams_build_and_push_break_glass_gen diff --git a/etc/evergreen_yml_components/variants/amazon/test_release.yml b/etc/evergreen_yml_components/variants/amazon/test_release.yml index a47bf4353c1..5094ee54b1b 100644 --- a/etc/evergreen_yml_components/variants/amazon/test_release.yml +++ b/etc/evergreen_yml_components/variants/amazon/test_release.yml @@ -496,7 +496,6 @@ buildvariants: # - name: .release_critical .requires_large_host !publish_packages !push !crypt_push # distros: # - amazon2023.3-arm64-large - - &enterprise-amazon2023-arm64-fuzzers-template <<: *enterprise-amazon2023-arm64-template name: enterprise-amazon2023-arm64-fuzzers diff --git a/evergreen/streams_build_js_engine.sh b/evergreen/streams_build_js_engine.sh index 717801d2ec5..e1d6d70644c 100644 --- a/evergreen/streams_build_js_engine.sh +++ b/evergreen/streams_build_js_engine.sh @@ -198,13 +198,16 @@ copy_js_engine() { local build_dir="$1" local target_dir="$2" - log "Copying JS engine production output from $build_dir/dist to $target_dir" + log "Copying JS engine production output to $target_dir" # Create target directory if it doesn't exist mkdir -p "$target_dir" # Copy only the dist folder contents with proper permissions - cp -r "$build_dir/dist"/* "$target_dir/" + cp -r "$build_dir/dist" "$target_dir/" + cp -r "$build_dir/node_modules" "$target_dir/" + cp -r "$build_dir/proto" "$target_dir/" + cp "$build_dir/package.json" "$target_dir/" chmod -R 755 "$target_dir" log "✓ JS engine production output copied successfully" diff --git a/src/mongo/scripting/BUILD.bazel b/src/mongo/scripting/BUILD.bazel index bbce1c52ee1..2fe00e4e884 100644 --- a/src/mongo/scripting/BUILD.bazel +++ b/src/mongo/scripting/BUILD.bazel @@ -20,6 +20,11 @@ mongo_cc_library( ], ) +idl_generator( + name = "config_gen", + src = "config.idl", +) + idl_generator( name = "deadline_monitor_gen", src = "deadline_monitor.idl", @@ -33,6 +38,7 @@ mongo_cc_library( "engine.cpp", "jsexception.cpp", "utils.cpp", + ":config_gen", ":deadline_monitor_gen", ], deps = [ diff --git a/src/mongo/scripting/config.idl b/src/mongo/scripting/config.idl new file mode 100644 index 00000000000..a4f993530c9 --- /dev/null +++ b/src/mongo/scripting/config.idl @@ -0,0 +1,39 @@ +# Copyright (C) 2025-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# . +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + +server_parameters: + enableExternalScripting: + description: "Enable external JavaScript execution using an external JS engine server" + set_at: startup + cpp_vartype: bool + cpp_varname: gEnableExternalScripting + default: false + redact: false diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index 270f28d0986..c3d8f80fd68 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -55,8 +55,8 @@ #include namespace mongo { -typedef unsigned long long ScriptingFunction; -typedef BSONObj (*NativeFunction)(const BSONObj& args, void* data); +using ScriptingFunction MONGO_MOD_PUBLIC = unsigned long long; +using NativeFunction MONGO_MOD_PUBLIC = BSONObj (*)(const BSONObj& args, void* data); typedef std::map FunctionCacheMap; class DBClientBase; @@ -67,7 +67,7 @@ struct MONGO_MOD_NEEDS_REPLACEMENT JSFile { const StringData source; }; -struct JSRegEx { +struct MONGO_MOD_PUBLIC JSRegEx { std::string pattern; std::string flags; @@ -76,7 +76,7 @@ struct JSRegEx { : pattern(std::move(pattern)), flags(std::move(flags)) {} }; -class MONGO_MOD_PUB Scope { +class MONGO_MOD_OPEN Scope { Scope(const Scope&) = delete; Scope& operator=(const Scope&) = delete; @@ -237,7 +237,7 @@ protected: enum class MONGO_MOD_PUB ExecutionEnvironment { Server, TestRunner }; -class MONGO_MOD_PUB ScriptEngine : public KillOpListenerInterface { +class MONGO_MOD_OPEN ScriptEngine : public KillOpListenerInterface { ScriptEngine(const ScriptEngine&) = delete; ScriptEngine& operator=(const ScriptEngine&) = delete; @@ -308,7 +308,11 @@ public: void interrupt(ClientLock&, OperationContext*) override {} void interruptAll(ServiceContextLock&) override {} - static std::string getInterpreterVersionString(); + /** + * Returns a string identifying the JavaScript interpreter implementation. + * For example: "MozJS", "ExternalJS", etc. + */ + virtual std::string getInterpreterVersionString() const = 0; protected: virtual Scope* createScope() = 0; @@ -324,6 +328,12 @@ bool hasJSReturn(const std::string& s); const char* jsSkipWhiteSpace(const char* raw); MONGO_MOD_PUB ScriptEngine* getGlobalScriptEngine(); -void setGlobalScriptEngine(ScriptEngine* impl); +MONGO_MOD_PUB void setGlobalScriptEngine(ScriptEngine* impl); +/** + * Returns true if external scripting is enabled. + * Default implementation returns false. + * Enterprise module provides an override that returns the IDL-controlled value. + */ +bool isExternalScriptingEnabled(); } // namespace mongo diff --git a/src/mongo/scripting/engine_none.cpp b/src/mongo/scripting/engine_none.cpp index a5370528d23..637e9c76db2 100644 --- a/src/mongo/scripting/engine_none.cpp +++ b/src/mongo/scripting/engine_none.cpp @@ -35,8 +35,4 @@ namespace mongo { void ScriptEngine::setup(ExecutionEnvironment environment) { // noop } - -std::string ScriptEngine::getInterpreterVersionString() { - return ""; -} } // namespace mongo diff --git a/src/mongo/scripting/mozjs/shell/engine.cpp b/src/mongo/scripting/mozjs/shell/engine.cpp index ef11e78e8e4..8292bdce838 100644 --- a/src/mongo/scripting/mozjs/shell/engine.cpp +++ b/src/mongo/scripting/mozjs/shell/engine.cpp @@ -32,9 +32,11 @@ #include "mongo/base/error_codes.h" #include "mongo/base/status.h" #include "mongo/db/operation_context.h" +#include "mongo/db/server_options.h" #include "mongo/db/service_context.h" #include "mongo/logv2/log.h" #include "mongo/platform/compiler.h" +#include "mongo/scripting/config_gen.h" #include "mongo/scripting/mozjs/shell/engine_gen.h" #include "mongo/scripting/mozjs/shell/implscope.h" #include "mongo/scripting/mozjs/shell/proxyscope.h" @@ -61,6 +63,10 @@ void DisableExtraThreads(); namespace mongo { +bool isExternalScriptingEnabled() { + return gEnableExternalScripting; +} + namespace { auto operationMozJSScopeBaseDecoration = OperationContext::declareDecoration(); @@ -71,6 +77,18 @@ void ScriptEngine::setup(ExecutionEnvironment environment) { return; } + // If gEnableExternalScripting is true, don't set up the MozJS engine. + if (isExternalScriptingEnabled()) { + if (!serverGlobalParams.quiet.load()) { + LOGV2_INFO(8972601, "External scripting is enabled. Not setting up MozJS engine."); + } + return; + } + + if (!serverGlobalParams.quiet.load()) { + LOGV2_INFO(8972602, "Setting up MozJS engine."); + } + setGlobalScriptEngine(new mozjs::MozJSScriptEngine(environment)); if (hasGlobalServiceContext()) { @@ -78,10 +96,6 @@ void ScriptEngine::setup(ExecutionEnvironment environment) { } } -std::string ScriptEngine::getInterpreterVersionString() { - return fmt::format("MozJS-{}", MOZJS_MAJOR_VERSION); -} - namespace mozjs { MozJSScriptEngine::MozJSScriptEngine(ExecutionEnvironment environment) @@ -157,6 +171,10 @@ void MozJSScriptEngine::setLoadPath(const std::string& loadPath) { _loadPath = loadPath; } +std::string MozJSScriptEngine::getInterpreterVersionString() const { + return fmt::format("MozJS-{}", MOZJS_MAJOR_VERSION); +} + void MozJSScriptEngine::registerOperation(OperationContext* opCtx, MozJSImplScope* scope) { LOGV2_DEBUG(22785, 2, diff --git a/src/mongo/scripting/mozjs/shell/engine.h b/src/mongo/scripting/mozjs/shell/engine.h index e0839e66646..913431f6734 100644 --- a/src/mongo/scripting/mozjs/shell/engine.h +++ b/src/mongo/scripting/mozjs/shell/engine.h @@ -75,6 +75,8 @@ public: std::string getLoadPath() const override; void setLoadPath(const std::string& loadPath) override; + std::string getInterpreterVersionString() const override; + void registerOperation(OperationContext* ctx, MozJSImplScope* scope); void unregisterOperation(OperationContext* opCtx);