SERVER-126784: Enable Remote Asset API External Artifact Caching for Releases (#53827) (#53908)

GitOrigin-RevId: c2196156370386f0a6befafd1b4735b36f472794

.bazelrc

@@ -516,8 +516,7 @@ common:macos -c dbg
 # Remote execution and caching is the default, but only mongodb employees will be able to access
 # the engflow cluster. External builders should use the --config=local option
 
-# Enable remote build execution:
+# Disable remote execution on Windows and MacOS (they cannot reach the engflow cluster):
-common:linux --remote_executor=grpcs://sodalite.cluster.engflow.com
 common:windows --remote_executor=
 common:macos --remote_executor=
 
@@ -525,7 +524,8 @@ common:macos --remote_executor=
 common:windows --//bazel/config:build_atlas=False
 common:macos --//bazel/config:build_atlas=False
 
-# Enable remote cache (also necessary for remote build execution):
+# Enable remote build execution and caching:
+common --remote_executor=grpcs://sodalite.cluster.engflow.com
 common --experimental_remote_downloader=grpcs://sodalite.cluster.engflow.com
 common --remote_cache=grpcs://sodalite.cluster.engflow.com
 common --bes_backend=grpcs://sodalite.cluster.engflow.com
@@ -594,27 +594,20 @@ common:no-remote-exec --jobs=auto
 --config=public-release
 common:public-release --//bazel/config:release=True
 common:public-release --//bazel/config:extensions_signature_verification_secure=True
+common:public-release --remote_instance_name=release
+common:public-release --bes_instance_name=release
+common:public-release --remote_upload_local_results=false
 common:public-release --stamp
 
-# Disable remote execution and caching for public releases
+# Disable remote execution for public releases but keep remote downloader for artifact caching
 --config=public-release-local
 common:public-release-local --config=public-release
 common:public-release-local --remote_executor=
-common:public-release-local --experimental_remote_downloader=
+common:public-release-local --modify_execution_info=.*=+no-cache
-common:public-release-local --remote_cache=
-common:public-release-local --bes_backend=
-common:public-release-local --bes_results_url=
-common:public-release-local --tls_client_certificate=
-common:public-release-local --tls_client_key=
-common:public-release-local --remote_cache_compression=false
-common:public-release-local --grpc_keepalive_time=0s
-common:public-release-local --legacy_important_outputs
 
 # Release configuration with rbe enabled
 --config=public-release-rbe
 common:public-release-rbe --config=public-release
-common:public-release-rbe --remote_instance_name=release
-common:public-release-rbe --bes_instance_name=release
 
 common --@rules_rust//rust/toolchain/channel=nightly
 common --@rules_rust//rust/settings:experimental_use_allocator_libraries_with_mangled_symbols=true

buildscripts/package_test.py

@@ -687,6 +687,46 @@ def validate_atlas(sources_text, edition, binfile):
             raise Exception(f"Failed to find atlas code in {edition} binary {binfile}.")
 
 
+def validate_no_remote_cache_or_execution(bep_json_path: str) -> None:
+    """Validate that the build did not use remote cache or remote execution.
+
+    Parses a Bazel Build Event Protocol (BEP) JSON file and checks that
+    --remote_executor was empty/unset and --modify_execution_info=.*=+no-cache
+    was set. The remote cache endpoint may still be configured (needed by the
+    remote downloader for artifact caching) as long as action caching is disabled.
+    """
+    logging.info("Validating no remote cache or execution in BEP file: %s", bep_json_path)
+    with open(bep_json_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            event = json.loads(line)
+            if "optionsParsed" not in event.get("id", {}):
+                continue
+            cmd_line = event.get("optionsParsed", {}).get("cmdLine", [])
+            remote_executor = ""
+            has_no_cache = False
+            for opt in cmd_line:
+                if opt.startswith("--remote_executor="):
+                    remote_executor = opt[len("--remote_executor=") :]
+                elif opt.startswith("--modify_execution_info=") and "no-cache" in opt:
+                    has_no_cache = True
+            if remote_executor:
+                raise Exception(
+                    f"Build used remote execution: --remote_executor={remote_executor}. "
+                    "Release builds must not use remote execution."
+                )
+            if not has_no_cache:
+                raise Exception(
+                    "Build did not disable action caching. "
+                    "Release builds must set --modify_execution_info=.*=+no-cache."
+                )
+            logging.info("Validated: no remote cache or remote execution detected in BEP")
+            return
+    raise Exception(f"No optionsParsed event found in BEP file: {bep_json_path}")
+
+
 def validate_no_libdwarf(sources_text, edition, binfile):
     if "third_party/libdwarf" in sources_text:
         raise Exception(f"Found LGPL code from libdwarf in {edition} binary {binfile}.")
@@ -781,6 +821,13 @@ branch_test_parser.add_argument(
     help="Evergreen task display name that owns the Packages artifact when using --evg-build-id.",
     default="package",
 )
+branch_test_parser.add_argument(
+    "--bep-json-file",
+    type=str,
+    help="Path to a Bazel Build Event Protocol JSON file. "
+    "Validates that no remote cache or remote execution was used to build the binaries.",
+    required=True,
+)
 args = parser.parse_args()
 
 if args.command == "release":
@@ -916,6 +963,9 @@ if args.command == "branch":
                 if p.returncode != 0:
                     raise Exception("GDB process exited non-zero!")
 
+    if os.environ.get("is_patch") != "true" or os.environ.get("is_release", "false") != "false":
+        validate_no_remote_cache_or_execution(args.bep_json_file)
+
 # If os is None we only want to do the tests specified in the arguments
 if args.command == "release":
     for dl in iterate_over_downloads():

evergreen/bazel_compile.sh

@@ -123,7 +123,9 @@ if [[ -n "${no_mongo_version}" ]]; then
 fi
 
 # Build flags line
-ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${MONGO_VERSION_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-}"
+BEP_FULL="build_events_full.json"
+BEP_OUT="build_events.json"
+ALL_FLAGS="--verbose_failures ${LOCAL_ARG} ${MONGO_VERSION_ARG} ${bazel_args:-} ${bazel_compile_flags:-} ${task_compile_flags:-} ${patch_compile_flags:-} --build_event_json_file=${BEP_FULL}"
 echo "${ALL_FLAGS}" >.bazel_build_flags
 
 # Save the entire bazel build invocation to attach to the task for re-running locally
@@ -138,6 +140,14 @@ RET=$?
 
 bazel_evergreen_shutils::write_last_engflow_link
 
+# Extract just the optionsParsed event from the full BEP JSON.
+# This single line (~few KB) is all package_test.py needs to verify
+# that remote cache and remote execution were not used.
+if [[ -f "${BEP_FULL}" ]]; then
+    grep '"optionsParsed"' "${BEP_FULL}" >"${BEP_OUT}" || true
+    rm -f "${BEP_FULL}"
+fi
+
 set -o errexit
 
 if [[ "$RET" -eq 124 ]]; then

evergreen/generate_evergreen_bazelrc.sh

@@ -55,8 +55,8 @@ uri="https://spruce.mongodb.com/task/${task_id:?}?execution=${execution:?}"
 
 echo "common --tls_client_certificate=./engflow.cert" >>.bazelrc.evergreen
 echo "common --tls_client_key=./engflow.key" >>.bazelrc.evergreen
-echo "common:public-release-rbe --tls_client_certificate=./.tmp/engflow_release.cert" >>.bazelrc.evergreen
+echo "common:public-release --tls_client_certificate=./.tmp/engflow_release.cert" >>.bazelrc.evergreen
-echo "common:public-release-rbe --tls_client_key=./.tmp/engflow_release.key" >>.bazelrc.evergreen
+echo "common:public-release --tls_client_key=./.tmp/engflow_release.key" >>.bazelrc.evergreen
 echo "common --bes_keywords=engflow:CiCdPipelineName=${build_variant:?}" >>.bazelrc.evergreen
 echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >>.bazelrc.evergreen
 echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >>.bazelrc.evergreen