From da6a8167cc4294a45fa91c7b8b63789bc19eb6ea Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Fri, 8 May 2026 10:16:43 -0400 Subject: [PATCH] SERVER-126227 Move monguard Endor Labs scan to Evergreen (#53419) GitOrigin-RevId: 3401ca36c7d9ca60114f0b26009c73880fb8f1e9 --- buildscripts/sbom/config.py | 6 ++++-- buildscripts/sbom/endorctl_scan.sh | 26 ++++++++++++++++++++++++++ buildscripts/sbom/generate_sbom.py | 15 +++++++++++++++ buildscripts/sbom/sbom_utils.py | 23 +++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100755 buildscripts/sbom/endorctl_scan.sh diff --git a/buildscripts/sbom/config.py b/buildscripts/sbom/config.py index 7e977e6f084..b3c8c93918f 100644 --- a/buildscripts/sbom/config.py +++ b/buildscripts/sbom/config.py @@ -40,8 +40,10 @@ for component in components_remove: # List of folders in src/third_party to exclude from SBOM generation warnings third_party_folders_remove = [ - "src/third_party/scripts", # this folder contains scripts related to the import process, but does not contain SBOM components itself - "src/third_party/private", # this is not a real third-party folder, but rather a place for MongoDB to store private forks of third-party code. The actual SBOM components in this folder are still included. + # this folder contains scripts related to the import process, but does not contain SBOM components itself + "src/third_party/scripts", + # this is not a real third-party folder, but rather a place for MongoDB to store private forks of third-party code. The actual SBOM components in this folder are still included. + "src/third_party/private", "src/third_party/boringssl_replacement", # this is an alias folder "src/third_party/wasmtime", # currently no targets depend on this ] diff --git a/buildscripts/sbom/endorctl_scan.sh b/buildscripts/sbom/endorctl_scan.sh new file mode 100755 index 00000000000..7ee641bc92a --- /dev/null +++ b/buildscripts/sbom/endorctl_scan.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Run an endorctl scan. All scan flags are driven by ENDORCTL_* environment variables. +# See: https://docs.endorlabs.com/developers-api/cli/environment-variables +# https://docs.endorlabs.com/developers-api/cli/commands/scan#options + +set -o errexit +set -o xtrace + +# Optional: set up a Rust toolchain when the scan requires cargo. +# Set RUST_SETUP_DIR to the path of a directory containing .evergreen/install-dependencies.sh +# (e.g. the monguard directory). The github_token env var must also be set for private +# 10gen dependency access. +if [ -n "${RUST_SETUP_DIR}" ]; then + pushd "${RUST_SETUP_DIR}" + export PROJECT_DIRECTORY="${PWD}" + source .evergreen/install-dependencies.sh rust + source .evergreen/env.sh + # env.sh sets CARGO_HOME inside the repo checkout (src/.cargo); override it to sit outside + # the repo root so endorctl does not scan the cargo package cache. + export CARGO_HOME="$(dirname "$(dirname "${RUST_SETUP_DIR}")")/.cargo" + source .evergreen/cargo-auth-env.sh + popd + cd "$(dirname "${RUST_SETUP_DIR}")" +fi + +"${ENDORCTL_PATH:-endorctl}" scan diff --git a/buildscripts/sbom/generate_sbom.py b/buildscripts/sbom/generate_sbom.py index 355873d3ac4..b336a318e83 100755 --- a/buildscripts/sbom/generate_sbom.py +++ b/buildscripts/sbom/generate_sbom.py @@ -31,6 +31,7 @@ from git import Commit, Repo from buildscripts.sbom.sbom_utils import ( add_component_dependsOn, add_component_property, + check_components_and_dependencies, check_metadata_sbom, convert_sbom_to_public, read_sbom_json_file, @@ -531,6 +532,18 @@ def main() -> None: print_banner("Pre-Processing Endor Labs SBOM") + ## deduplicate components by bom-ref ## + seen_bom_refs: set[str] = set() + unique_components = [] + for component in endor_bom["components"]: + bom_ref = component.get("bom-ref", "") + if bom_ref in seen_bom_refs: + logger.info("ENDOR SBOM PRE-PROCESS: removing duplicate bom-ref %s", bom_ref) + else: + seen_bom_refs.add(bom_ref) + unique_components.append(component) + endor_bom["components"] = unique_components + ## remove uneeded components ## # [list]endor_components_remove is defined in config.py @@ -584,6 +597,7 @@ def main() -> None: if os.path.exists(sbom_metadata_path): meta_bom = read_sbom_json_file(sbom_metadata_path) + check_components_and_dependencies(meta_bom, sbom_metadata_path) else: logger.error("No SBOM metadata file at '%s'. This is fatal.", sbom_metadata_path) sys.exit(1) @@ -1130,6 +1144,7 @@ def main() -> None: ) license_entry["license"]["id"] = new + check_components_and_dependencies(meta_bom, sbom_out_internal_path) write_sbom_json_file(meta_bom, sbom_out_internal_path) # Load the previous public SBOM to track its serialNumber/version independently diff --git a/buildscripts/sbom/sbom_utils.py b/buildscripts/sbom/sbom_utils.py index 002cca9c5c5..f097a894b57 100644 --- a/buildscripts/sbom/sbom_utils.py +++ b/buildscripts/sbom/sbom_utils.py @@ -113,6 +113,29 @@ def check_metadata_sbom(meta_bom: dict) -> None: ) +def check_components_and_dependencies(sbom: dict, label: str = "") -> None: + """Warn if .components[].bom-ref and .dependencies[].ref are not in one-to-one correspondence.""" + prefix = f"{label}: " if label else "" + component_refs = {c["bom-ref"] for c in sbom.get("components", [])} + dependency_refs = {d["ref"] for d in sbom.get("dependencies", [])} + + in_components_not_deps = component_refs - dependency_refs + in_deps_not_components = dependency_refs - component_refs + + if in_components_not_deps: + logger.warning( + "%sCOMPONENTS/DEPENDENCIES MISMATCH: components with no matching dependency ref: %s", + prefix, + sorted(in_components_not_deps), + ) + if in_deps_not_components: + logger.warning( + "%sCOMPONENTS/DEPENDENCIES MISMATCH: dependency refs with no matching component: %s", + prefix, + sorted(in_deps_not_components), + ) + + def convert_sbom_to_public(sbom_dict: dict): """Remove internal-only properties and components from SBOM"""