SERVER-116378 Add new variant for publishing spidermonkey side wasm build (#51150)

GitOrigin-RevId: 2dbb3c31ffd51f218d3bfced8383ac4f3e4e8e20

MODULE.bazel

@@ -64,9 +64,9 @@ spidermonkey_wasi_repo = use_repo_rule(
 
 spidermonkey_wasi_repo(
     name = "spidermonkey_wasi",
-    sha256 = "f528e5730721559781aef47d355079da63185588bb14ebb6bf5266d4aa5e3eb0",
+    sha256 = "e89f9a9959555bb08444412600e6a4594125e15d68145f97c6d88e6395be1d6b",
     urls = [
-        "https://mdb-build-public.s3.us-east-1.amazonaws.com/wasm_scratch/spidermonkey-wasip2-release.tar.gz",
+        "https://mdb-build-public.s3.amazonaws.com/spidermonkey-wasm/FIREFOX_140_6_0esr_RELEASE/69d013e89378880007aaa010/spidermonkey-wasip2-release.tar.gz",
     ] * 5,
 )
 
@@ -485,5 +485,6 @@ spidermonkey_repo = use_repo_rule(
 spidermonkey_repo(
     name = "spidermonkey",
     repository_file = "//src/mongo/scripting/mozjs/wasm/spider-monkey:spider-monkey-repository",
+    sha256_file = "//src/mongo/scripting/mozjs/wasm/spider-monkey:spider-monkey-sha256sum",
     version_file = "//src/mongo/scripting/mozjs/wasm/spider-monkey:spider-monkey-version",
 )

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -1387,6 +1387,44 @@ tasks:
             - "https://mdb-build-public.s3.us-east-1.amazonaws.com/profiling_data/pgo/mongod_${revision}_${build_arch}_gcc_lto_pgo_${version}.tgz"
             - "${branch_name}"
             - "SERVER-110427-${revision}"
+
+  - name: build_and_upload_spidermonkey_wasm
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: //src/mongo/scripting/mozjs/wasm:spidermonkey_wasip2_dist_release_from_source
+          bazel_args: >-
+            --config=evg
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - "-c"
+            - |
+              cd src
+              sm_version=$(cat src/mongo/scripting/mozjs/wasm/spider-monkey/spider-monkey-version | tr -d '[:space:]')
+              echo "spidermonkey_version: $sm_version" > spidermonkey_expansions.yml
+      - command: expansions.update
+        params:
+          file: src/spidermonkey_expansions.yml
+      - func: "f_expansions_write"
+      - command: s3.put
+        params:
+          aws_key: ${aws_key_build}
+          aws_secret: ${aws_secret_build}
+          local_file: src/bazel-bin/src/mongo/scripting/mozjs/wasm/spidermonkey-wasip2-release.tar.gz
+          remote_file: spidermonkey-wasm/${spidermonkey_version}/spidermonkey-wasip2-release.tar.gz
+          bucket: mdb-build-public
+          permissions: public-read
+          # Do not remove this, it is not safe to overwrite identical versions as they might not be hash perfect
+          skip_existing: true
+          content_type: application/gzip
+          display_name: SpiderMonkey WASI P2 Release
+
   - name: publish_crypt_packages
     run_on: rhel8.7-small
     tags: [
@@ -1494,6 +1532,11 @@ task_groups:
     tasks:
       - rseq_kernel_compatibility_check_jstest
 
+  - <<: *compile_bazel_task_group_template
+    name: build_and_upload_spidermonkey_wasm_TG
+    tasks:
+      - build_and_upload_spidermonkey_wasm
+
   # We want to run these sequentially to put less load on the cluster
   - <<: *compile_task_group_template
     name: hydrate_bazel_profile_TG

etc/evergreen_yml_components/variants/misc/misc.yml

@@ -135,3 +135,20 @@ buildvariants:
     stepback: false
     tasks:
       - name: upload_sbom_via_silkbomb_if_changed
+
+  # This variant is used to create a new spidermonkey-wasip2-release.tar.gz file
+  # Its not included as part of the regular build due to its size, complexity and
+  # non-hermeticity.
+  - name: spidermonkey-wasm-compile
+    display_name: "SpiderMonkey WASM Compile & Upload"
+    tags: ["assigned_to_jira_team_server_query_integration"]
+    patch_only: true
+    run_on:
+      - amazon2023-arm64-latest-large
+    expansions:
+      compile_variant: spidermonkey-wasm-compile
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+    tasks:
+      - name: build_and_upload_spidermonkey_wasm_TG

src/mongo/scripting/mozjs/wasm/BUILD.bazel

@@ -13,6 +13,8 @@ exports_files(
 )
 
 # Build SpiderMonkey tarball from source (manual, for re-uploading to S3).
+# If you want to try it locally replace spidermonkey_wasi module url with
+# file://<local_path>
 genrule(
     name = "spidermonkey_wasip2_dist_release_from_source",
     srcs = [
@@ -28,6 +30,7 @@ genrule(
       export SPIDER_MACH_PATH="$(location @spidermonkey//:mach)"
       export RUST_SHIMS_LIB_RS="$(location support/rust_shims/src/lib.rs)"
       export CARGO_TEMPLATE_PATH="$(location support/rust_shims/Cargo.toml.template)"
+      export HOST_CC="$(CC)"
       export OUTPUT="$@"
       bash $(location scripts/build_spidermonkey_wasip2.sh)
     """,
@@ -36,6 +39,7 @@ genrule(
         "manual",
         "no-sandbox",
     ],
+    toolchains = ["@bazel_tools//tools/cpp:current_cc_toolchain"],
     tools = [
         "@wasi_sdk//:bin",
     ],

src/mongo/scripting/mozjs/wasm/scripts/build_spidermonkey_wasip2.sh

@@ -53,8 +53,16 @@ if [ -z "$REAL_HOME" ]; then
     fi
 fi
 export HOME="$REAL_HOME"
+export PATH="$HOME/.cargo/bin:$PATH"
 export MOZ_FETCHES_DIR="$HOME/.mozbuild"
 
+# Install rustup + cargo if not available (e.g. fresh CI hosts).
+if ! command -v cargo >/dev/null 2>&1; then
+    echo "cargo not found; installing rustup..." >&2
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+    export PATH="$HOME/.cargo/bin:$PATH"
+fi
+
 # --- Locate WASI SDK ---
 if [ -z "${WASI_SDK_PATH:-}" ]; then
     WASI_CLANG=""
@@ -114,8 +122,22 @@ export CC="$WASI_SDK_PATH/bin/wasm32-wasip2-clang"
 export CXX="$WASI_SDK_PATH/bin/wasm32-wasip2-clang++"
 export AR="$WASI_SDK_PATH/bin/llvm-ar"
 export RANLIB="$WASI_SDK_PATH/bin/llvm-ranlib"
-export HOST_CC="$(command -v clang)"
-export HOST_CXX="$(command -v clang++)"
+# HOST_CC is passed in from the Bazel genrule via $(CC). It may be a relative
+# path under Bazel's execroot, so resolve it to an absolute path.
+if [ -z "${HOST_CC:-}" ]; then
+    echo "ERROR: HOST_CC not set; expected from Bazel \$(CC) toolchain variable." >&2
+    exit 1
+fi
+case "$HOST_CC" in
+/*) ;; # already absolute
+*) HOST_CC="$EXECROOT/$HOST_CC" ;;
+esac
+case "$HOST_CC" in
+*gcc) export HOST_CXX="${HOST_CC/%gcc/g++}" ;;
+*clang) export HOST_CXX="${HOST_CC}++" ;;
+*cc) export HOST_CXX="${HOST_CC/%cc/c++}" ;;
+*) export HOST_CXX="${HOST_CC}++" ;;
+esac
 
 # Enable per-function/data sections so the linker's --gc-sections can remove
 # unused code.  Also hide internal symbols to aid dead-code elimination.
@@ -128,13 +150,22 @@ export RUST_TARGET=wasm32-wasip2
 
 # Configure expects a Rust stdlib for wasm32-wasip2 to be installed.
 # This is a local (non-hermetic) build step by design (tags = local/no-sandbox).
-if [ -x "$HOME/.cargo/bin/rustup" ]; then
-    "$HOME/.cargo/bin/rustup" target add wasm32-wasip2 || true
-elif command -v rustup >/dev/null 2>&1; then
+if command -v rustup >/dev/null 2>&1; then
     rustup target add wasm32-wasip2 || true
 fi
 
 export MOZCONFIG="$MOZCONFIG"
+
+# cbindgen is required by SpiderMonkey's configure step.
+if ! command -v cbindgen >/dev/null 2>&1; then
+    echo "cbindgen not found; installing via cargo..." >&2
+    if ! command -v cargo >/dev/null 2>&1; then
+        echo "ERROR: cargo not found; cannot install cbindgen" >&2
+        exit 1
+    fi
+    cargo install cbindgen
+fi
+
 cd "$SRC_ROOT_REAL"
 python3 "./mach" -v --no-interactive build
 

src/mongo/scripting/mozjs/wasm/spider-monkey/BUILD.bazel

@@ -2,5 +2,6 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files([
     "spider-monkey-repository",
+    "spider-monkey-sha256sum",
     "spider-monkey-version",
 ])

src/mongo/scripting/mozjs/wasm/spider-monkey/README.md

@@ -0,0 +1,33 @@
+# Updating SpiderMonkey
+
+## Steps
+
+1. **Update the version and checksum files:**
+
+   - Set `spider-monkey-version` to the new Firefox/SpiderMonkey release tag
+     (e.g. `FIREFOX_141_0esr_RELEASE`).
+   - Update `spider-monkey-sha256sum` with the SHA-256 of the corresponding
+     GitHub source archive. You can compute it with:
+     ```
+     curl -sL "https://github.com/mozilla-firefox/firefox/archive/refs/tags/<TAG>.tar.gz" | sha256sum
+     ```
+
+2. **Run the CI variant in a patch build:**
+
+   - Create an Evergreen patch that includes your changes.
+   - Run the `spidermonkey-wasm-compile` variant. This builds the
+     `spidermonkey_wasip2_dist_release_from_source` target and uploads the
+     resulting tarball to S3 under
+     `spidermonkey-wasm/<spidermonkey_version>/`.
+
+3. **Get the new tarball URL:**
+
+   - Go to the patch build's **Files** tab in the Evergreen UI.
+   - Copy the S3 URL for `spidermonkey-wasip2-release.tar.gz`.
+
+4. **Update the spidermonkey_wasi dependency:**
+
+   - Update the `spidermonkey_wasi` module's URL and SHA-256 to point at the
+     newly uploaded tarball.
+
+5. **Open a PR** linking the patch build that produced the tarball.

src/mongo/scripting/mozjs/wasm/spider-monkey/spider-monkey-sha256sum

@@ -0,0 +1 @@
+4b9e12d5840356ad0451e89b3e96bd84846516b3fa5c18ed88430cc9f8525358

src/mongo/scripting/mozjs/wasm/spidermonkey_repo.bzl

@@ -17,11 +17,14 @@ def _basename_no_git(url):
 def _spidermonkey_repository_impl(rctx):
     repo_url = _strip(rctx.read(rctx.attr.repository_file))
     version = _strip(rctx.read(rctx.attr.version_file))
+    sha256 = _strip(rctx.read(rctx.attr.sha256_file))
 
     if not repo_url:
         fail("spidermonkey_repository: repository_file was empty")
     if not version:
         fail("spidermonkey_repository: version_file was empty")
+    if not sha256:
+        fail("spidermonkey_repository: sha256_file was empty")
 
     # Normalize.
     if repo_url.endswith("/"):
@@ -36,13 +39,9 @@ def _spidermonkey_repository_impl(rctx):
     archive_url = "{}/archive/refs/tags/{}.tar.gz".format(repo_url, version)
     strip_prefix = "{}-{}".format(repo_name, version)
 
-    if not rctx.attr.sha256:
-        # buildifier: disable=print
-        print("WARNING: spidermonkey_repository: sha256 not set for %s; download integrity will not be verified." % archive_url)
-
     rctx.download_and_extract(
         url = archive_url,
-        sha256 = rctx.attr.sha256,
+        sha256 = sha256,
         stripPrefix = strip_prefix,
     )
 
@@ -73,8 +72,7 @@ spidermonkey_repository = repository_rule(
         # Labels in the main workspace that contain the repo URL and tag/version.
         "repository_file": attr.label(mandatory = True, allow_single_file = True),
         "version_file": attr.label(mandatory = True, allow_single_file = True),
-        # Optional, but recommended for hermeticity. If empty, Bazel will not verify.
-        "sha256": attr.string(default = ""),
+        "sha256_file": attr.label(mandatory = True, allow_single_file = True),
     },
     doc = "Downloads SpiderMonkey/Firefox source from a repo+tag stored in files, and exposes `:mach` and `:srcs`.",
 )