SERVER-115951: Adds External JS Server integration (#43960)

GitOrigin-RevId: 241578dc158ad14b67a0dda1f4dda20d46d218ba

.github/CODEOWNERS

@@ -2386,6 +2386,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 # The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams/aspio/OWNERS.yml
 /src/mongo/db/modules/enterprise/jstests/streams/aspio/**/* @10gen/streams-engine @svc-auto-approve-bot
 
+# The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams/externaljs/OWNERS.yml
+/src/mongo/db/modules/enterprise/jstests/streams/externaljs/**/* @10gen/streams-engine @svc-auto-approve-bot
+
 # The following patterns are parsed from ./src/mongo/db/modules/enterprise/jstests/streams_kafka/OWNERS.yml
 /src/mongo/db/modules/enterprise/jstests/streams_kafka/**/* @10gen/streams-engine @svc-auto-approve-bot
 

.prettierignore

@@ -64,3 +64,5 @@ bazel-*
 
 # Streams specific
 src/mongo/db/modules/enterprise/src/streams/third_party/mongocxx/dist
+# asp-js-engine is an external module copied during Docker builds
+asp-js-engine

buildscripts/resmokeconfig/suites/streams_externaljs.yml

@@ -0,0 +1,31 @@
+test_kind: js_test
+
+selector:
+  roots:
+    - src/mongo/db/modules/*/jstests/streams/externaljs/*.js
+
+executor:
+  config:
+    shell_options:
+      global_vars:
+        TestData:
+          # Prevent auto-execution of imported test modules. Tests in this suite
+          # use containerized mongostream and must control when tests run.
+          skipDefaultRun: true
+  fixture:
+    class: ReplicaSetFixture
+    mongod_options:
+      bind_ip_all: ""
+      set_parameters:
+        enableTestCommands: 1
+        featureFlagStreams: true
+        diagnosticDataCollectionEnabled: false
+        # ExternalJS server parameters
+        # These can be overridden via --setParameter on the command line
+        enableExternalScripting: true
+        jsBinPath: "/usr/bin/node"
+        jsSvrPath: "/app/externaljs/dist/server.js"
+        jsSvrPort: 50051
+        jsSvrWrkDir: "/tmp/"
+        jsSvrStartTimeoutMs: 10000
+    num_nodes: 1

etc/evergreen_yml_components/definitions.yml

@@ -1254,6 +1254,23 @@ functions:
         OTEL_PARENT_ID: ${otel_parent_id}
         OTEL_COLLECTOR_DIR: "../build/OTelTraces/"
 
+  "execute resmoke tests with asp js engine":
+    &execute_resmoke_tests_with_asp_js_engine_token
+    command: subprocess.exec
+    display_name: "execute resmoke tests with asp js engine"
+    type: test
+    params:
+      binary: bash
+      args:
+        - "./src/evergreen/resmoke_tests_execute.sh"
+      env:
+        OTEL_TRACE_ID: ${otel_trace_id}
+        OTEL_PARENT_ID: ${otel_parent_id}
+        OTEL_COLLECTOR_DIR: "../build/OTelTraces/"
+        # Path to the asp-js-engine module cloned via Evergreen modules
+        ASP_JS_ENGINE_PATH: ${workdir}/asp-js-engine
+        HAS_JS_ENGINE: ${HAS_JS_ENGINE|}
+
   "execute resmoke tests via bazel sh": &execute_resmoke_tests_via_bazel_sh
     command: subprocess.exec
     display_name: "execute resmoke tests via bazel sh"
@@ -1658,6 +1675,75 @@ functions:
     - *check_run_tests_infrastructure_failure
     - *check_resmoke_failure
 
+  # Run streams tests that require access to the private asp-js-engine repo
+  # Clone the asp-js-engine module and restore git history/tags so resmoke's git describe works
+  # Re-fetch and extract binaries after git clone because the clone overwrites src/ (including tarballs and dist-test/)
+  # Copy asp-js-engine from src/ to workdir where resmoke expects it (ASP_JS_ENGINE_PATH=${workdir}/asp-js-engine)
+  "run streams tests":
+    - *f_expansions_write
+    - *git_get_shallow_streams_project
+    - *restore_git_history_and_tags
+    # Copy asp-js-engine module from src/ to workdir where ASP_JS_ENGINE_PATH expects it
+    - command: subprocess.exec
+      display_name: "copy asp-js-engine to workdir"
+      params:
+        binary: bash
+        args:
+          - "-c"
+          - |
+            set -o errexit
+            set -o verbose
+            # git.get_project clones into src/, so asp-js-engine module is at src/asp-js-engine/asp-js-engine
+            # ASP_JS_ENGINE_PATH expects it at ${workdir}/asp-js-engine with package.json at root
+            rm -rf asp-js-engine
+            cp -r src/asp-js-engine/asp-js-engine asp-js-engine
+    - *fetch_binaries
+    - *fetch_binaries_zstd
+    - *fetch_tgz_binary_shas
+    - *fetch_and_verify_binaries_sha
+    - *fetch_and_verify_binaries_sha_zstd
+    - *fetch_jstestshell
+    - *verify_jstestshell_sha
+    - *extract_binaries
+    - *extract_jstestshell
+    - *configure_evergreen_api_credentials
+    - *determine_task_timeout
+    - *update_task_timeout_expansions
+    - *f_expansions_write
+    - *update_task_timeout
+    - *f_expansions_write
+    - *set_code_coverage_expansion
+    - *f_expansions_write
+    - command: expansions.update
+      params:
+        env:
+          CEDAR_USER: ${cedar_user}
+          CEDAR_API_KEY: ${cedar_api_key}
+        updates:
+          - key: aws_key_remote
+            value: ${mongodatafiles_aws_key}
+          - key: aws_profile_remote
+            value: mongodata_aws
+          - key: aws_secret_remote
+            value: ${mongodatafiles_aws_secret}
+    - *f_expansions_write
+    - *set_up_remote_credentials
+    - *f_expansions_write
+    - *determine_resmoke_jobs
+    - *update_resmoke_jobs_expansions
+    - *f_expansions_write
+    - *configure_evergreen_api_credentials
+    - *sign_macos_dev_binaries
+    - *multiversion_exclude_tags_generate
+    - *assume_ecr_role
+    - *fetch_module_images
+    - *execute_resmoke_tests_with_asp_js_engine_token
+      # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
+      # directly actionable. We use type=setup rather than type=system or type=test for this command
+    # because we don't intend for any human to look at this failure.
+    - *check_run_tests_infrastructure_failure
+    - *check_resmoke_failure
+
   "run benchmark tests":
     - *f_expansions_write
     - *configure_evergreen_api_credentials

etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml

@@ -323,7 +323,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -338,7 +338,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -353,7 +353,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -368,7 +368,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -383,7 +383,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -398,7 +398,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
@@ -413,10 +413,26 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 
+  - <<: *task_template
+    name: streams_externaljs
+    tags:
+      [
+        "assigned_to_jira_team_streams",
+        "default",
+        "streams_release_test",
+        "requires_extra_system_deps",
+      ]
+    commands:
+      - func: "do setup"
+      - func: "run streams tests"
+        vars:
+          resmoke_jobs_max: 1
+          HAS_JS_ENGINE: "true"
+
   - <<: *task_template
     name: streams_aspio_pubsub
     tags:
@@ -428,7 +444,7 @@ tasks:
       ]
     commands:
       - func: "do setup"
-      - func: "run tests"
+      - func: "run streams tests"
         vars:
           resmoke_jobs_max: 1
 

etc/evergreen_yml_components/variants/amazon/streams/streams_release.yml

@@ -60,6 +60,7 @@ buildvariants:
       - name: streams_aspio_iceberg_3
       - name: streams_aspio_iceberg_4
       - name: streams_aspio_iceberg_5
+      - name: streams_externaljs
       - name: streams_aspio_pubsub
       - name: streams_build_and_push_gen
       - name: streams_build_and_push_break_glass_gen
@@ -118,6 +119,7 @@ buildvariants:
       - name: streams_aspio_iceberg_3
       - name: streams_aspio_iceberg_4
       - name: streams_aspio_iceberg_5
+      - name: streams_externaljs
       - name: streams_aspio_pubsub
       - name: streams_build_and_push_gen
       - name: streams_build_and_push_break_glass_gen

etc/evergreen_yml_components/variants/amazon/test_release.yml

@@ -496,7 +496,6 @@ buildvariants:
   #     - name: .release_critical .requires_large_host !publish_packages !push !crypt_push
   #       distros:
   #         - amazon2023.3-arm64-large
-
   - &enterprise-amazon2023-arm64-fuzzers-template
     <<: *enterprise-amazon2023-arm64-template
     name: enterprise-amazon2023-arm64-fuzzers

evergreen/streams_build_js_engine.sh

@@ -198,13 +198,16 @@ copy_js_engine() {
     local build_dir="$1"
     local target_dir="$2"
 
-    log "Copying JS engine production output from $build_dir/dist to $target_dir"
+    log "Copying JS engine production output to $target_dir"
 
     # Create target directory if it doesn't exist
     mkdir -p "$target_dir"
 
     # Copy only the dist folder contents with proper permissions
-    cp -r "$build_dir/dist"/* "$target_dir/"
+    cp -r "$build_dir/dist" "$target_dir/"
+    cp -r "$build_dir/node_modules" "$target_dir/"
+    cp -r "$build_dir/proto" "$target_dir/"
+    cp "$build_dir/package.json" "$target_dir/"
     chmod -R 755 "$target_dir"
 
     log "✓ JS engine production output copied successfully"

src/mongo/scripting/BUILD.bazel

@@ -20,6 +20,11 @@ mongo_cc_library(
     ],
 )
 
+idl_generator(
+    name = "config_gen",
+    src = "config.idl",
+)
+
 idl_generator(
     name = "deadline_monitor_gen",
     src = "deadline_monitor.idl",
@@ -33,6 +38,7 @@ mongo_cc_library(
         "engine.cpp",
         "jsexception.cpp",
         "utils.cpp",
+        ":config_gen",
         ":deadline_monitor_gen",
     ],
     deps = [

src/mongo/scripting/config.idl

@@ -0,0 +1,39 @@
+# Copyright (C) 2025-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+global:
+    cpp_namespace: "mongo"
+
+server_parameters:
+    enableExternalScripting:
+        description: "Enable external JavaScript execution using an external JS engine server"
+        set_at: startup
+        cpp_vartype: bool
+        cpp_varname: gEnableExternalScripting
+        default: false
+        redact: false

src/mongo/scripting/engine.h

@@ -55,8 +55,8 @@
 #include <boost/optional/optional.hpp>
 
 namespace mongo {
-typedef unsigned long long ScriptingFunction;
-typedef BSONObj (*NativeFunction)(const BSONObj& args, void* data);
+using ScriptingFunction MONGO_MOD_PUBLIC = unsigned long long;
+using NativeFunction MONGO_MOD_PUBLIC = BSONObj (*)(const BSONObj& args, void* data);
 typedef std::map<std::string, ScriptingFunction> FunctionCacheMap;
 
 class DBClientBase;
@@ -67,7 +67,7 @@ struct MONGO_MOD_NEEDS_REPLACEMENT JSFile {
     const StringData source;
 };
 
-struct JSRegEx {
+struct MONGO_MOD_PUBLIC JSRegEx {
     std::string pattern;
     std::string flags;
 
@@ -76,7 +76,7 @@ struct JSRegEx {
         : pattern(std::move(pattern)), flags(std::move(flags)) {}
 };
 
-class MONGO_MOD_PUB Scope {
+class MONGO_MOD_OPEN Scope {
     Scope(const Scope&) = delete;
     Scope& operator=(const Scope&) = delete;
 
@@ -237,7 +237,7 @@ protected:
 
 enum class MONGO_MOD_PUB ExecutionEnvironment { Server, TestRunner };
 
-class MONGO_MOD_PUB ScriptEngine : public KillOpListenerInterface {
+class MONGO_MOD_OPEN ScriptEngine : public KillOpListenerInterface {
     ScriptEngine(const ScriptEngine&) = delete;
     ScriptEngine& operator=(const ScriptEngine&) = delete;
 
@@ -308,7 +308,11 @@ public:
     void interrupt(ClientLock&, OperationContext*) override {}
     void interruptAll(ServiceContextLock&) override {}
 
-    static std::string getInterpreterVersionString();
+    /**
+     * Returns a string identifying the JavaScript interpreter implementation.
+     * For example: "MozJS", "ExternalJS", etc.
+     */
+    virtual std::string getInterpreterVersionString() const = 0;
 
 protected:
     virtual Scope* createScope() = 0;
@@ -324,6 +328,12 @@ bool hasJSReturn(const std::string& s);
 const char* jsSkipWhiteSpace(const char* raw);
 
 MONGO_MOD_PUB ScriptEngine* getGlobalScriptEngine();
-void setGlobalScriptEngine(ScriptEngine* impl);
+MONGO_MOD_PUB void setGlobalScriptEngine(ScriptEngine* impl);
 
+/**
+ * Returns true if external scripting is enabled.
+ * Default implementation returns false.
+ * Enterprise module provides an override that returns the IDL-controlled value.
+ */
+bool isExternalScriptingEnabled();
 }  // namespace mongo

src/mongo/scripting/engine_none.cpp

@@ -35,8 +35,4 @@ namespace mongo {
 void ScriptEngine::setup(ExecutionEnvironment environment) {
     // noop
 }
-
-std::string ScriptEngine::getInterpreterVersionString() {
-    return "";
-}
 }  // namespace mongo

src/mongo/scripting/mozjs/shell/engine.cpp

@@ -32,9 +32,11 @@
 #include "mongo/base/error_codes.h"
 #include "mongo/base/status.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
 #include "mongo/logv2/log.h"
 #include "mongo/platform/compiler.h"
+#include "mongo/scripting/config_gen.h"
 #include "mongo/scripting/mozjs/shell/engine_gen.h"
 #include "mongo/scripting/mozjs/shell/implscope.h"
 #include "mongo/scripting/mozjs/shell/proxyscope.h"
@@ -61,6 +63,10 @@ void DisableExtraThreads();
 
 namespace mongo {
 
+bool isExternalScriptingEnabled() {
+    return gEnableExternalScripting;
+}
+
 namespace {
 auto operationMozJSScopeBaseDecoration =
     OperationContext::declareDecoration<mozjs::MozJSImplScope*>();
@@ -71,6 +77,18 @@ void ScriptEngine::setup(ExecutionEnvironment environment) {
         return;
     }
 
+    // If gEnableExternalScripting is true, don't set up the MozJS engine.
+    if (isExternalScriptingEnabled()) {
+        if (!serverGlobalParams.quiet.load()) {
+            LOGV2_INFO(8972601, "External scripting is enabled. Not setting up MozJS engine.");
+        }
+        return;
+    }
+
+    if (!serverGlobalParams.quiet.load()) {
+        LOGV2_INFO(8972602, "Setting up MozJS engine.");
+    }
+
     setGlobalScriptEngine(new mozjs::MozJSScriptEngine(environment));
 
     if (hasGlobalServiceContext()) {
@@ -78,10 +96,6 @@ void ScriptEngine::setup(ExecutionEnvironment environment) {
     }
 }
 
-std::string ScriptEngine::getInterpreterVersionString() {
-    return fmt::format("MozJS-{}", MOZJS_MAJOR_VERSION);
-}
-
 namespace mozjs {
 
 MozJSScriptEngine::MozJSScriptEngine(ExecutionEnvironment environment)
@@ -157,6 +171,10 @@ void MozJSScriptEngine::setLoadPath(const std::string& loadPath) {
     _loadPath = loadPath;
 }
 
+std::string MozJSScriptEngine::getInterpreterVersionString() const {
+    return fmt::format("MozJS-{}", MOZJS_MAJOR_VERSION);
+}
+
 void MozJSScriptEngine::registerOperation(OperationContext* opCtx, MozJSImplScope* scope) {
     LOGV2_DEBUG(22785,
                 2,

src/mongo/scripting/mozjs/shell/engine.h

@@ -75,6 +75,8 @@ public:
     std::string getLoadPath() const override;
     void setLoadPath(const std::string& loadPath) override;
 
+    std::string getInterpreterVersionString() const override;
+
     void registerOperation(OperationContext* ctx, MozJSImplScope* scope);
     void unregisterOperation(OperationContext* opCtx);