SERVER-126227 Move monguard Endor Labs scan to Evergreen (#53419)

GitOrigin-RevId: 3401ca36c7d9ca60114f0b26009c73880fb8f1e9

buildscripts/sbom/config.py

@@ -40,8 +40,10 @@ for component in components_remove:
 
 # List of folders in src/third_party to exclude from SBOM generation warnings
 third_party_folders_remove = [
-    "src/third_party/scripts",  # this folder contains scripts related to the import process, but does not contain SBOM components itself
-    "src/third_party/private",  # this is not a real third-party folder, but rather a place for MongoDB to store private forks of third-party code. The actual SBOM components in this folder are still included.
+    # this folder contains scripts related to the import process, but does not contain SBOM components itself
+    "src/third_party/scripts",
+    # this is not a real third-party folder, but rather a place for MongoDB to store private forks of third-party code. The actual SBOM components in this folder are still included.
+    "src/third_party/private",
     "src/third_party/boringssl_replacement",  # this is an alias folder
     "src/third_party/wasmtime",  # currently no targets depend on this
 ]

buildscripts/sbom/endorctl_scan.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Run an endorctl scan. All scan flags are driven by ENDORCTL_* environment variables.
+# See: https://docs.endorlabs.com/developers-api/cli/environment-variables
+#      https://docs.endorlabs.com/developers-api/cli/commands/scan#options
+
+set -o errexit
+set -o xtrace
+
+# Optional: set up a Rust toolchain when the scan requires cargo.
+# Set RUST_SETUP_DIR to the path of a directory containing .evergreen/install-dependencies.sh
+# (e.g. the monguard directory). The github_token env var must also be set for private
+# 10gen dependency access.
+if [ -n "${RUST_SETUP_DIR}" ]; then
+    pushd "${RUST_SETUP_DIR}"
+    export PROJECT_DIRECTORY="${PWD}"
+    source .evergreen/install-dependencies.sh rust
+    source .evergreen/env.sh
+    # env.sh sets CARGO_HOME inside the repo checkout (src/.cargo); override it to sit outside
+    # the repo root so endorctl does not scan the cargo package cache.
+    export CARGO_HOME="$(dirname "$(dirname "${RUST_SETUP_DIR}")")/.cargo"
+    source .evergreen/cargo-auth-env.sh
+    popd
+    cd "$(dirname "${RUST_SETUP_DIR}")"
+fi
+
+"${ENDORCTL_PATH:-endorctl}" scan

buildscripts/sbom/generate_sbom.py

@@ -31,6 +31,7 @@ from git import Commit, Repo
 from buildscripts.sbom.sbom_utils import (
     add_component_dependsOn,
     add_component_property,
+    check_components_and_dependencies,
     check_metadata_sbom,
     convert_sbom_to_public,
     read_sbom_json_file,
@@ -531,6 +532,18 @@ def main() -> None:
 
     print_banner("Pre-Processing Endor Labs SBOM")
 
+    ## deduplicate components by bom-ref ##
+    seen_bom_refs: set[str] = set()
+    unique_components = []
+    for component in endor_bom["components"]:
+        bom_ref = component.get("bom-ref", "")
+        if bom_ref in seen_bom_refs:
+            logger.info("ENDOR SBOM PRE-PROCESS: removing duplicate bom-ref %s", bom_ref)
+        else:
+            seen_bom_refs.add(bom_ref)
+            unique_components.append(component)
+    endor_bom["components"] = unique_components
+
     ## remove uneeded components ##
     # [list]endor_components_remove is defined in config.py
 
@@ -584,6 +597,7 @@ def main() -> None:
 
     if os.path.exists(sbom_metadata_path):
         meta_bom = read_sbom_json_file(sbom_metadata_path)
+        check_components_and_dependencies(meta_bom, sbom_metadata_path)
     else:
         logger.error("No SBOM metadata file at '%s'. This is fatal.", sbom_metadata_path)
         sys.exit(1)
@@ -1130,6 +1144,7 @@ def main() -> None:
                     )
                     license_entry["license"]["id"] = new
 
+    check_components_and_dependencies(meta_bom, sbom_out_internal_path)
     write_sbom_json_file(meta_bom, sbom_out_internal_path)
 
     # Load the previous public SBOM to track its serialNumber/version independently

buildscripts/sbom/sbom_utils.py

@@ -113,6 +113,29 @@ def check_metadata_sbom(meta_bom: dict) -> None:
                 )
 
 
+def check_components_and_dependencies(sbom: dict, label: str = "") -> None:
+    """Warn if .components[].bom-ref and .dependencies[].ref are not in one-to-one correspondence."""
+    prefix = f"{label}: " if label else ""
+    component_refs = {c["bom-ref"] for c in sbom.get("components", [])}
+    dependency_refs = {d["ref"] for d in sbom.get("dependencies", [])}
+
+    in_components_not_deps = component_refs - dependency_refs
+    in_deps_not_components = dependency_refs - component_refs
+
+    if in_components_not_deps:
+        logger.warning(
+            "%sCOMPONENTS/DEPENDENCIES MISMATCH: components with no matching dependency ref: %s",
+            prefix,
+            sorted(in_components_not_deps),
+        )
+    if in_deps_not_components:
+        logger.warning(
+            "%sCOMPONENTS/DEPENDENCIES MISMATCH: dependency refs with no matching component: %s",
+            prefix,
+            sorted(in_deps_not_components),
+        )
+
+
 def convert_sbom_to_public(sbom_dict: dict):
     """Remove internal-only properties and components from SBOM"""