From 4be6f775aa69a7fff263c152ac3dfd04b4aaa79e Mon Sep 17 00:00:00 2001 From: ekovalets Date: Tue, 14 Oct 2025 14:17:26 -0700 Subject: [PATCH] SERVER-110425: SBOM upload and SCA automation 8.2 (#41877) GitOrigin-RevId: 40c0e396749b7f323644b76d4f6ad752d4773f53 --- etc/evergreen_lint.yml | 3 + .../tasks/misc_tasks.yml | 53 ++++ .../variants/rhel/test_dev.yml | 13 + .../functions/upload_sbom_via_silkbomb.py | 260 ++++++++++++++++++ .../write_aws_creds_to_silkbomb_env_file.sh | 8 + 5 files changed, 337 insertions(+) create mode 100644 evergreen/functions/upload_sbom_via_silkbomb.py create mode 100644 evergreen/write_aws_creds_to_silkbomb_env_file.sh diff --git a/etc/evergreen_lint.yml b/etc/evergreen_lint.yml index 7f3109d922d..95fcadb0b40 100644 --- a/etc/evergreen_lint.yml +++ b/etc/evergreen_lint.yml @@ -94,6 +94,9 @@ rules: # https://github.com/10gen/mothra/blob/main/mothra/teams/et.yaml - assigned_to_jira_team_streams + + # https://github.com/10gen/mothra/blob/main/mothra/teams/security.yaml + - assigned_to_jira_team_platsec_server min_num_of_tags: 1 max_num_of_tags: 1 # Every task should have required selection tag diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 54876a6a3f5..9150f16c364 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1917,6 +1917,59 @@ tasks: GITHUB_REPO: ${github_repo} GITHUB_TOKEN: ${github_token} + - name: upload_sbom_via_silkbomb_if_changed + allowed_requesters: ["commit", "patch"] + tags: ["auxiliary", "assigned_to_jira_team_platsec_server"] + exec_timeout_secs: 600 # 10 minute timeout + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - command: ec2.assume_role + display_name: Assume Silkbomb IAM role + params: + role_arn: arn:aws:iam::119629040606:role/silkbomb + - func: "f_expansions_write" + - command: subprocess.exec + display_name: Write temporary AWS credentials to Silkbomb environment file + params: + binary: bash + args: + - "src/evergreen/write_aws_creds_to_silkbomb_env_file.sh" + include_expansions_in_env: + [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + - command: ec2.assume_role + display_name: Assume DevProd Platforms ECR readonly IAM role + params: + role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - github_token + args: + - "./src/evergreen/run_python_script.sh" + - "evergreen/functions/upload_sbom_via_silkbomb.py" + - "--run" + env: + REQUESTER: ${requester} + BRANCH_NAME: ${branch_name} + GITHUB_ORG: ${github_org} + GITHUB_REPO: ${github_repo} + CONTAINER_COMMAND: podman # podman or docker + CONTAINER_IMAGE: 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 + CONTAINER_ENV_FILES: ${workdir}/silkbomb.env + WORKING_DIR: ${workdir} + SBOM_REPO_PATH: sbom.json + LOCAL_REPO_PATH: src + - name: check_for_noexcept allowed_requesters: ["github_pr"] tags: diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev.yml b/etc/evergreen_yml_components/variants/rhel/test_dev.yml index 048b8b05ffd..30813add668 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev.yml @@ -362,3 +362,16 @@ buildvariants: - name: sharding_pqs_fallback_gen - name: sharding_pqs_hints_gen - name: sharding_pqs_index_filters_gen + + - name: upload-sbom-if-changed + display_name: "Upload SBOM if changed" + allowed_requesters: ["commit"] + activate: true + paths: + - "sbom.json" + tags: ["auxiliary", "assigned_to_jira_team_platsec_server"] + run_on: + - rhel8.8-small + stepback: false + tasks: + - name: upload_sbom_via_silkbomb_if_changed diff --git a/evergreen/functions/upload_sbom_via_silkbomb.py b/evergreen/functions/upload_sbom_via_silkbomb.py new file mode 100644 index 00000000000..9ed36693233 --- /dev/null +++ b/evergreen/functions/upload_sbom_via_silkbomb.py @@ -0,0 +1,260 @@ +import pathlib +import subprocess +import sys + +import typer +from git import Repo +from typing_extensions import Annotated + +app = typer.Typer( + help="Checks for SBOM file changes in a PR and uploads it to Kondukto if changed.", + add_completion=False, +) + + +def get_changed_files_from_latest_commit(local_repo_path: str, branch_name: str = "master") -> dict: + try: + repo = Repo(local_repo_path) + + if branch_name not in repo.heads: + raise ValueError(f"Branch '{branch_name}' does not exist in the repository.") + + last_commit = repo.heads[branch_name].commit + title = last_commit.summary + commit_hash = last_commit.hexsha + + # If the last commit has no parents, it means it's the first commit in the repo + if not last_commit.parents: + files = [item.path for item in last_commit.tree.traverse()] + else: + # Comparing the last commit with its parent to find changed files + files = [file.a_path for file in last_commit.diff(last_commit.parents[0])] + + return {"title": title, "hash": commit_hash, "files": files} + except Exception as e: + print(f"Error retrieving changed files: {e}") + raise e + + +def upload_sbom_via_silkbomb( + sbom_repo_path: str, + workdir: str, + local_repo_path: str, + repo_name: str, + branch_name: str, + creds_file_path: pathlib.Path, + container_command: str, + container_image: str, + timeout_seconds: int = 60 * 5, +): + container_options = ["--pull=always", "--platform=linux/amd64", "--rm"] + container_env_files = ["--env-file", str(creds_file_path.resolve())] + container_volumes = ["-v", f"{workdir}:/workdir"] + silkbomb_command = "augment" # it augment first and uses upload command + silkbomb_args = [ + "--sbom-in", + f"/workdir/{local_repo_path}/{sbom_repo_path}", + "--branch", + branch_name, + "--repo", + repo_name, + ] + + command = [ + container_command, + "run", + *container_options, + *container_env_files, + *container_volumes, + container_image, + silkbomb_command, + *silkbomb_args, + ] + + aws_region = "us-east-1" + ecr_registry_url = ( + "901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb" + ) + + print(f"Attempting to authenticate to AWS ECR registry '{ecr_registry_url}'...") + try: + login_cmd = f"aws ecr get-login-password --region {aws_region} | {container_command} login --username AWS --password-stdin {ecr_registry_url}" + subprocess.run( + login_cmd, + shell=True, + check=True, + text=True, + capture_output=True, + timeout=timeout_seconds, + ) + print("ECR authentication successful.") + except FileNotFoundError: + print( + f"Error: A required command was not found. Please ensure AWS CLI and '{container_command}' are installed and in your PATH." + ) + raise + except subprocess.TimeoutExpired as e: + print( + f"Error: Command timed out after {timeout_seconds} seconds. Please check Evergreen network state and try again." + ) + raise e + except subprocess.CalledProcessError as e: + print(f"Error during ECR authentication:\n--- STDERR ---\n{e.stderr}") + raise + + try: + print(f"Running command: {' '.join(command)}") + subprocess.run(command, check=True, text=True, capture_output=True, timeout=timeout_seconds) + print("Updated sbom.json file upload via Silkbomb successful!") + except FileNotFoundError as e: + print(f"Error: '{container_command}' command not found.") + raise e + except subprocess.TimeoutExpired as e: + print( + f"Error: Command timed out after {timeout_seconds} seconds. Please check Evergreen network state and try again." + ) + raise e + except subprocess.CalledProcessError as e: + print( + f"Error during container execution:\n--- STDOUT ---\n{e.stdout}\n--- STDERR ---\n{e.stderr}" + ) + raise e + + +# TODO (SERVER-109205): Add Slack Alerts for failures +@app.command() +def run( + github_org: Annotated[ + str, + typer.Option(..., envvar="GITHUB_ORG", help="Name of the github organization (e.g. 10gen)"), + ], + github_repo: Annotated[ + str, typer.Option(..., envvar="GITHUB_REPO", help="Repo name in 'owner/repo' format.") + ], + local_repo_path: Annotated[ + str, + typer.Option(..., envvar="LOCAL_REPO_PATH", help="Path to the local git repository."), + ], + branch_name: Annotated[ + str, + typer.Option(..., envvar="BRANCH_NAME", help="The head branch (e.g., the PR branch name)."), + ], + sbom_repo_path: Annotated[ + str, + typer.Option( + ..., + "--sbom-in", + envvar="SBOM_REPO_PATH", + help="Path to the SBOM file to check and upload.", + ), + ] = "sbom.json", + requester: Annotated[ + str, + typer.Option( + ..., + envvar="REQUESTER", + help="The entity requesting the run (e.g., 'github_merge_queue').", + ), + ] = "", + container_command: Annotated[ + str, + typer.Option( + ..., envvar="CONTAINER_COMMAND", help="Container engine to use ('podman' or 'docker')." + ), + ] = "podman", + container_image: Annotated[ + str, typer.Option(..., envvar="CONTAINER_IMAGE", help="Silkbomb container image.") + ] = "901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0", + creds_file: Annotated[ + pathlib.Path, + typer.Option( + ..., envvar="CONTAINER_ENV_FILES", help="Path for the temporary credentials file." + ), + ] = pathlib.Path("kondukto_credentials.env"), + workdir: Annotated[ + str, typer.Option(..., envvar="WORKING_DIR", help="Path for the container volumes.") + ] = "/workdir", + dry_run: Annotated[ + bool, typer.Option("--dry-run/--run", help="Check for changes without uploading.") + ] = True, + check_sbom_file_change: Annotated[ + bool, typer.Option("--check-sbom-file-change", help="Check for changes to the SBOM file.") + ] = False, +): + if requester != "commit" and not dry_run: + print(f"Skipping: Run can only be triggered for 'commit', but requester was '{requester}'.") + sys.exit(0) + + major_branches = ["v7.0", "v8.0", "v8.2", "master"] # Only major branches that MongoDB supports + if False and branch_name not in major_branches: + print(f"Skipping: Branch '{branch_name}' is not a major branch. Exiting.") + sys.exit(0) + + repo_path = pathlib.Path(f"{workdir}/{local_repo_path}") + sbom_path = pathlib.Path(f"{repo_path}/{sbom_repo_path}") + if not sbom_path.resolve().exists(): + print(f"Error: SBOM file not found at path: {str(sbom_path.resolve())}") + sys.exit(1) + + try: + sbom_file_changed = True + if check_sbom_file_change: + commit_changed_files = get_changed_files_from_latest_commit(repo_path, branch_name) + if commit_changed_files: + print( + f"Latest commit '{commit_changed_files['title']}' ({commit_changed_files['hash']}) in branch '{branch_name}' has the following changed files:" + ) + print(f"{commit_changed_files['files']}") + else: + print( + f"No changed files found in the commit '{commit_changed_files['title']}' ({commit_changed_files['hash']}) in branch '{branch_name}'. Exiting without upload." + ) + sys.exit(0) + + print(f"Checking for changes to file: {sbom_path} ({sbom_repo_path})") + + sbom_file_changed = sbom_repo_path in commit_changed_files["files"] + + if sbom_file_changed: + print(f"File '{sbom_path}' was modified. Initiating upload...") + else: + print(f"File '{sbom_repo_path}' was not modified. Nothing to upload.") + + if not dry_run and sbom_file_changed: + upload_sbom_via_silkbomb( + sbom_repo_path=sbom_repo_path, + workdir=workdir, + local_repo_path=local_repo_path, + repo_name=f"{github_org}/{github_repo}", + branch_name=branch_name, + creds_file_path=creds_file, + container_command=container_command, + container_image=container_image, + ) + else: + print("--dry-run enabled, skipping upload.") + print( + f"File '{sbom_repo_path}'" + + (" was modified." if sbom_file_changed else " was not modified.") + ) + + if dry_run: + print("Upload metadata:") + print(f" SBOM Path: {sbom_path}") + print(f" Repo Name: '{github_org}/{github_repo}'") + print(f" Repo Branch: '{branch_name}'") + print(f" Container Command: {container_command}") + print(f" Container Image: {container_image}") + print(f" Workdir: {workdir}") + if check_sbom_file_change: + print( + f"Latest commit '{commit_changed_files['title']}' ({commit_changed_files['hash']})" + ) + + except Exception as e: + print(f"Exception during script execution: {e}") + sys.exit(1) + + +if __name__ == "__main__": + app() diff --git a/evergreen/write_aws_creds_to_silkbomb_env_file.sh b/evergreen/write_aws_creds_to_silkbomb_env_file.sh new file mode 100644 index 00000000000..441deb310fc --- /dev/null +++ b/evergreen/write_aws_creds_to_silkbomb_env_file.sh @@ -0,0 +1,8 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cat <"${workdir}/silkbomb.env" +AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} +AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} +AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} +EOF