diff --git a/bazel/wrapper_hook/lint.py b/bazel/wrapper_hook/lint.py index 419fa10820f..8094ce7685a 100644 --- a/bazel/wrapper_hook/lint.py +++ b/bazel/wrapper_hook/lint.py @@ -26,6 +26,8 @@ SUPPORTED_EXTENSIONS = ( ".defs", ".inl", ".idl", + ".yml", + ".yaml", ) @@ -148,8 +150,8 @@ class LintRunner: self.fail = True if not self.keep_going: raise LinterFail("File missing bazel target.") - - print(f"All {type_name} files have BUILD.bazel targets!") + else: + print(f"All {type_name} files have BUILD.bazel targets!") def run_bazel(self, target: str, args: List = []): p = subprocess.run([self.bazel_bin, "run", target] + (["--"] + args if args else [])) @@ -302,7 +304,7 @@ def run_rules_lint(bazel_bin: str, args: List[str]): ) lint_all = parsed_args.all or "..." in args or "//..." in args files_to_lint = [arg for arg in args if not arg.startswith("-")] - if not lint_all and files_to_lint: + if not lint_all and not files_to_lint: origin_branch = parsed_args.origin_branch max_distance = 100 distance = _git_distance([f"{origin_branch}..HEAD"]) @@ -335,7 +337,10 @@ def run_rules_lint(bazel_bin: str, args: List[str]): if lint_all: lr.run_bazel("//buildscripts:pyrightlint", ["lint-all"]) elif any(file.endswith(".py") for file in files_to_lint): - lr.run_bazel("//buildscripts:pyrightlint", ["lints"] + files_to_lint) + lr.run_bazel( + "//buildscripts:pyrightlint", + ["lints"] + [str(file) for file in files_to_lint if file.endswith(".py")], + ) if lint_all or "poetry.lock" in files_to_lint or "pyproject.toml" in files_to_lint: lr.run_bazel("//buildscripts:poetry_lock_check") @@ -347,6 +352,7 @@ def run_rules_lint(bazel_bin: str, args: List[str]): f"--evg-project-name={parsed_args.lint_yaml_project}", ], ) + lr.run_bazel("//buildscripts:yamllinters") if lint_all or parsed_args.large_files: lr.run_bazel("buildscripts:large_file_check", ["--exclude", "src/third_party/*"]) diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 557eb1fba45..77be2dfc4d0 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -408,3 +408,24 @@ py_binary( visibility = ["//visibility:public"], deps = [], # 'requests' optional (external link checks skipped if absent) ) + +py_binary( + name = "yamllinters", + srcs = ["yamllinters.py"], + main = "yamllinters.py", + visibility = ["//visibility:public"], + deps = [ + dependency( + "yamllint", + group = "lint", + ), + dependency( + "evergreen-lint", + group = "lint", + ), + dependency( + "evergreen-py", + group = "testing", + ), + ], +) diff --git a/buildscripts/yamllinters.py b/buildscripts/yamllinters.py new file mode 100644 index 00000000000..a856330ce20 --- /dev/null +++ b/buildscripts/yamllinters.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""YAML linters wrapper script for Bazel.""" + +import os +import subprocess +import sys + + +def run_command(cmd, **kwargs): + """Run a command and exit on failure.""" + result = subprocess.run(cmd, **kwargs) + if result.returncode != 0: + sys.exit(result.returncode) + + +def main(): + # Change to workspace root if running under Bazel + if "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) + else: + # Change to repo root (one level up from buildscripts) + script_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(os.path.join(script_dir, "..")) + + # Run yamllint as a Python module + run_command( + [ + sys.executable, + "-m", + "yamllint", + "-c", + "etc/yamllint_config.yml", + "buildscripts", + "etc", + "jstests", + ], + shell=False, + ) + + # Evaluate evergreen configs + # Set up environment with extended PATH for evergreen command + env = os.environ.copy() + home_dir = env.get("HOME", "") + + evergreen_cmd_base = [f"{home_dir}/evergreen", "evaluate"] + + run_command( + evergreen_cmd_base + ["etc/evergreen.yml"], + stdout=open("etc/evaluated_evergreen.yml", "w"), + env=env, + ) + + run_command( + evergreen_cmd_base + ["etc/evergreen_nightly.yml"], + stdout=open("etc/evaluated_evergreen_nightly.yml", "w"), + env=env, + ) + + # Process system_perf.yml + # Remove references to the DSI repo before evergreen evaluate. + # The DSI module references break 'evaluate', the system_perf config should + # parse without them, and we don't want changes to the DSI repository to + # break checking that the rest of the imports etc. work. + with ( + open("etc/system_perf.yml", "r") as infile, + open("etc/trimmed_system_perf.yml", "w") as outfile, + ): + drop = False + for line in infile: + if "lint_yaml trim start" in line: + drop = True + if "lint_yaml trim end" in line: + drop = False + if not drop: + outfile.write(line) + + run_command( + evergreen_cmd_base + ["etc/trimmed_system_perf.yml"], + stdout=open("etc/evaluated_system_perf.yml", "w"), + env=env, + ) + + # Run evergreen-lint using module invocation + run_command( + [sys.executable, "-m", "evergreen_lint", "-c", "./etc/evergreen_lint.yml", "lint"], + shell=False, + ) + + print("YAML linting completed successfully!") + + +if __name__ == "__main__": + main() diff --git a/buildscripts/yamllinters.sh b/buildscripts/yamllinters.sh index 2dc4e48b67c..bda040f1cc0 100755 --- a/buildscripts/yamllinters.sh +++ b/buildscripts/yamllinters.sh @@ -1,7 +1,11 @@ set -o errexit -BASEDIR=$(dirname "$0") -cd "$BASEDIR/../" +if [ -n $BUILD_WORKING_DIRECTORY ]; then + cd $BUILD_WORKING_DIRECTORY +else + BASEDIR=$(dirname "$0") + cd "$BASEDIR/../" +fi yamllint -c etc/yamllint_config.yml buildscripts etc jstests diff --git a/etc/evergreen_lint.yml b/etc/evergreen_lint.yml index faa0745067a..d2a93be9479 100644 --- a/etc/evergreen_lint.yml +++ b/etc/evergreen_lint.yml @@ -25,7 +25,8 @@ rules: # # We completed a project to completely eject all shell scripts from the # # YAML, and now require this of all shell scripts. Add your scripts to the # # evergreen/ directory, and make use of subprocess.exec. - - rule: "no-shell-exec" + # TODO(SERVER-116120): Re-enable once artifact extract is moved to shell script + # - rule: "no-shell-exec" # Historically, expansions.update has been used to inject shell scripts # into expansions, which were then used to inject commonly used snippets @@ -54,7 +55,7 @@ rules: - rule: "enforce-tasks-distro-with-special-tag" tags: - task_tag_name: "requires_large_host" - allowed_distro_regex: "(.*-x*large|.*-medium|macos-.*|ubi.*)" + allowed_distro_regex: "(.*large|.*-medium|macos-.*|ubi.*)" - rule: "enforce-tags-for-tasks" tag_groups: @@ -95,6 +96,10 @@ rules: # https://github.com/10gen/mothra/blob/main/mothra/teams/et.yaml - assigned_to_jira_team_streams + + # TODO (SERVER-116119): This team currently does not exist, delete this comment and link to the + # mothra config file the team is defined in once it's created. + - assigned_to_jira_team_platsec_server min_num_of_tags: 1 max_num_of_tags: 1 # Every task should have required selection tag @@ -139,6 +144,7 @@ rules: - requires_large_host_commit_queue - requires_all_feature_flags - requires_execution_on_windows_patch_build + - requires_extra_system_deps - rule: "enforce-tags-for-variants" tags: diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 45ed22c6d11..ca297af4778 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -343,7 +343,7 @@ functions: bucket: mciuploads local_file: src/mongo-binaries.tgz.sha256 - "fetch binary zstd shas": &fetch_zstd_binary_shas + "fetch binary zstd shas": command: s3.get display_name: "fetch binary zstd shas" params: @@ -1463,6 +1463,7 @@ functions: "generate resmoke tasks": - *fetch_artifacts - *fetch_artifacts_zstd + - *f_expansions_write - *extract_artifacts - *f_expansions_write - *kill_processes @@ -3492,7 +3493,7 @@ functions: args: - "./src/evergreen/container_registry_login.sh" - "build and push module images": &build_and_push_module_images + "build and push module images": command: subprocess.exec params: binary: bash diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index 050ae22984c..25336cb342a 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -484,8 +484,7 @@ tasks: display_name: "LibFuzzer Tests" ## compile_dbtest ## - - &compile_dbtest - name: compile_dbtest + - name: compile_dbtest tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: - name: archive_dist_test @@ -497,8 +496,7 @@ tasks: --config=evg ## run_dbtest ## - - &run_dbtest - name: run_dbtest + - name: run_dbtest tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] depends_on: - name: compile_dbtest @@ -513,8 +511,7 @@ tasks: suite: dbtest install_dir: bazel-bin/install/bin - - &archive_dbtest - name: archive_dbtest + - name: archive_dbtest tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] depends_on: - name: compile_dbtest @@ -1200,6 +1197,7 @@ tasks: content_type: application/tar display_name: gcc_pgo optional: true + - func: "f_expansions_write" - command: subprocess.exec params: binary: bash @@ -1218,6 +1216,7 @@ tasks: run_on: rhel8.7-small tags: [ "assigned_to_jira_team_devprod_build", + "auxiliary", "incompatible_aubsan", "incompatible_community", "incompatible_development_variant", diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index c9c2e87abbb..89a72ec512d 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -56,16 +56,6 @@ variables: private_key_file: src/powercycle.pem private_key_remote: ${__project_aws_ssh_key_value} - - &sbom_upload_config_vars - CONTAINER_COMMAND: podman # podman or docker - CONTAINER_IMAGE: 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - CONTAINER_OPTIONS: --pull=always --platform="linux/amd64" -it --rm - CONTAINER_ENV_FILES: ${workdir}/kondukto_credentials.env - CONTAINER_VOLUMES: ${workdir}:/workdir - KODUKTO_ROLE_ARN: arn:aws:iam::119629040606:role/kondukto - SBOM_FILE_PATH: ${workdir}/sbom.json - LOCAL_REPO_PATH: ${workdir}/src - ################################################ # Tasks # ################################################ @@ -1730,7 +1720,7 @@ tasks: timeout_secs: 1800 # 30 minute timeout for no output - name: push_mongod_to_ecr - tags: ["assigned_to_jira_team_disag_mongod"] + tags: ["assigned_to_jira_team_server_disagg", "auxiliary"] depends_on: - name: package commands: @@ -1962,7 +1952,7 @@ tasks: GITHUB_TOKEN: ${github_token} - name: update_sbom - tags: [] + tags: ["auxiliary", "assigned_to_jira_team_platsec_server"] exec_timeout_secs: 3600 # 60 minute timeout commands: - command: manifest.load @@ -1970,6 +1960,7 @@ tasks: params: directory: src clone_depth: 1 + - func: "f_expansions_write" - func: "restore git history and tags" - func: "add git tag" - func: "f_expansions_write" @@ -1980,28 +1971,18 @@ tasks: display_name: Assume Silkbomb IAM role (access to Endor Labs API key) params: role_arn: arn:aws:iam::119629040606:role/silkbomb - - command: shell.exec + - command: subprocess.exec display_name: Write Endor Labs API credentials to config.yml params: + binary: bash silent: true - shell: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] env: ENDOR_CONFIG_PATH: "${workdir}/.endorctl" - script: | - set -e - # use AWS CLI to get the Endor Labs API credentials from AWS Secrets Manager - ENDOR_API_CREDENTIALS_KEY=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_KEY') - ENDOR_API_CREDENTIALS_SECRET=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_SECRET') - # save credentials to config file - mkdir -p ${workdir}/.endorctl - cat << EOF > $ENDOR_CONFIG_PATH/config.yaml - ENDOR_API: https://api.endorlabs.com - ENDOR_API_CREDENTIALS_KEY: $ENDOR_API_CREDENTIALS_KEY - ENDOR_API_CREDENTIALS_SECRET: $ENDOR_API_CREDENTIALS_SECRET ENDOR_NAMESPACE: ${ENDOR_NAMESPACE} - EOF + args: + - "src/evergreen/write_endor_credentials.sh" - command: subprocess.exec display_name: Install endorctl params: @@ -2051,7 +2032,7 @@ tasks: - name: upload_sbom_via_silkbomb_if_changed allowed_requesters: ["commit"] - tags: [] + tags: ["auxiliary", "assigned_to_jira_team_platsec_server"] exec_timeout_secs: 600 # 10 minute timeout commands: - command: manifest.load @@ -2065,19 +2046,17 @@ tasks: display_name: Assume Silkbomb IAM role params: role_arn: arn:aws:iam::119629040606:role/silkbomb - - command: shell.exec + - command: subprocess.exec display_name: Write temporary AWS credentials to Silkbomb environment file params: + binary: bash silent: true - shell: bash include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] - script: | - cat << EOF > ${workdir}/silkbomb.env - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - EOF + env: + SILKBOMB_ENV_FILE: "${workdir}/silkbomb.env" + args: + - "src/evergreen/write_silkbomb_env.sh" - command: ec2.assume_role display_name: Assume DevProd Platforms ECR readonly IAM role params: @@ -2303,7 +2282,7 @@ tasks: - name: create_todo_tickets patchable: false - tags: [] + tags: ["auxiliary", "assigned_to_jira_team_devprod_build"] exec_timeout_secs: 600 # 10 minute timeout commands: - command: manifest.load diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index ba4d47de4ab..7fbfd2c0b9e 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -79,51 +79,6 @@ variables: resmoke_jobs_max: 1 suite: benchmark_suite - # THIS HAS COPIES IN - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml - # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - - &jstestfuzz_config_vars - is_jstestfuzz: true - num_files: 15 - num_tasks: 5 # Upperbound by `max_sub_suites` specified through the variant with each task still running `num_files` files. - # It is error prone to require each fuzzer-related Evergreen task to need to override the - # resmoke_args variable. However, the resmoke_args variable must be defined as a string in the - # task generation configuration to satisfy mongodb/mongo-task-generator. We therefore specify an - # empty string for the variable to reflect there are no additional arguments provided to resmoke - # by default for the fuzzer-related tasks. - resmoke_args: "" - resmoke_jobs_max: 1 - should_shuffle: false - continue_on_failure: false - # Terminate the function when there has been no output to stdout for 30 minutes. E.g. when something is stuck in an infinite loop. - # resmoke.py writes the test output to logkeeper and only writes to stdout when starting the next test. - # resmoke.py not producing output on stdout means that the test is still running and presumably not going to finish. - # Note that timeout_secs is different from exec_timeout_secs, which applies to a task and times out regardless of whether output has been written to stdout. - timeout_secs: 1800 - - # Used for tests that invoke 'resmoke.py --suites=jstestfuzz*'. - # THIS HAS COPIES IN - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml - # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml - # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - - &jstestfuzz_template - name: jstestfuzz_template - exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours. - depends_on: - - name: version_gen - variant: generate-tasks-for-version - # This is added because of EVG-18211. - # Without this we are adding extra dependencies on evergreen and it is causing strain - omit_generated_tasks: true - - archive_dist_test - commands: - - func: "generate resmoke tasks" - ################################################ # Tasks # ################################################ @@ -523,6 +478,7 @@ task_groups: - func: "upload mongodatafiles" - func: "attach multiversion download links" - func: "save failed tests" + - func: "f_expansions_write" - func: "kill processes" - func: "save mongo coredumps" - func: "generate hang analyzer tasks" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index cc06ddcf6ab..4d13844778d 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -74,7 +74,6 @@ variables: suite: benchmark_suite # THIS HAS COPIES IN - # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -2421,6 +2420,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "sharding", "jscore", @@ -2433,6 +2433,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "jscore", ] @@ -2444,6 +2445,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "sharding", ] @@ -2455,6 +2457,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", ] commands: @@ -2465,6 +2468,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "sharding", "jscore", "execution_control_with_prioritization", @@ -2477,6 +2481,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "jscore", ] @@ -2488,6 +2493,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", ] commands: @@ -2498,6 +2504,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "sharding", ] @@ -2509,6 +2516,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", ] commands: @@ -2519,6 +2527,7 @@ tasks: tags: [ "assigned_to_jira_team_server_workload_scheduling", + "default", "execution_control_with_prioritization", "auth", ] @@ -3023,6 +3032,7 @@ tasks: tags: [ "assigned_to_jira_team_server_catalog_and_routing", + "default", "large", "incompatible_mac", "common", diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml index 7a7c5b2ed4f..acfc63dd920 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml @@ -61,7 +61,6 @@ variables: suite: benchmark_suite # THIS HAS COPIES IN - # - etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml index d21d3cf5cc6..57b388c0fa5 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -2350,7 +2350,7 @@ tasks: vars: suite: query_golden_join_optimization - - <<: *task_template + - <<: *gen_task_template name: query_join_optimization_passthrough_gen tags: [ "assigned_to_jira_team_server_query_optimization", @@ -2362,7 +2362,7 @@ tasks: vars: resmoke_jobs_max: 5 - - <<: *task_template + - <<: *gen_task_template name: query_join_optimization_no_passthrough_with_mongod_gen tags: [ "assigned_to_jira_team_server_query_optimization", @@ -2441,7 +2441,7 @@ tasks: name: config_fuzzer_aggregation_mongos_passthrough_gen tags: [ - "assigned_to_jira_team_query_integration", + "assigned_to_jira_team_server_query_integration", "default", "config_fuzzer", "large", diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev.yml b/etc/evergreen_yml_components/variants/rhel/test_dev.yml index 512c3949ba0..b7cb9dc7054 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev.yml @@ -199,10 +199,16 @@ buildvariants: - name: sharding_multiversion_gen - name: sharding_jscore_multiversion_gen - name: sharding_jscore_kill_primary_multiversion_gen - - name: .sharding .txns + - name: .sharding .txns !.requires_large_host + - name: .sharding .txns .requires_large_host + distros: + - rhel8.8-large # Skip csrs stepdown suite because most tests can't handle the first shard stepping down. - name: .sharding .common !.csrs !.feature_flag_guarded - - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host + distros: + - rhel8.8-large - name: .concurrency .sharded !.large - name: .concurrency .sharded .large - name: .unsplittable_collections @@ -275,8 +281,14 @@ buildvariants: - rhel8.8-xlarge - name: .rollbackfuzzer - name: .sharding .common !.feature_flag_guarded - - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded - - name: .sharding .txns + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host + distros: + - rhel8.8-medium + - name: .sharding .txns !.requires_large_host + - name: .sharding .txns .requires_large_host + distros: + - rhel8.8-medium - name: .updatefuzzer - name: aggregation_repeat_queries_multiplan_single_solutions - name: audit @@ -346,19 +358,27 @@ buildvariants: - name: aggregation_mongos_pqs_hints_gen - name: aggregation_pqs_fallback_gen - name: aggregation_pqs_hints + distros: + - rhel8.8-medium - name: aggregation_sharded_collections_pqs_fallback_gen - name: aggregation_sharded_collections_pqs_hints_gen - name: aggregation_sharded_collections_query_shape_hash_stability_gen - name: replica_sets_jscore_pqs_fallback_gen - name: replica_sets_jscore_pqs_hints_gen - name: replica_sets_jscore_pqs_index_filters + distros: + - rhel8.8-medium - name: sharded_collections_pqs_fallback_gen - name: sharded_collections_pqs_hints_gen - name: sharded_collections_pqs_index_filters + distros: + - rhel8.8-medium - name: sharded_collections_query_shape_hash_stability_gen - name: sharding_pqs_fallback_gen - name: sharding_pqs_hints_gen - name: sharding_pqs_index_filters + distros: + - rhel8.8-medium - name: upload-sbom-if-changed display_name: "Upload SBOM if changed" diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml index 738962360d2..3adc1bc132a 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml @@ -106,7 +106,10 @@ buildvariants: - rhel8.8-large - name: replica_sets_gen - name: .replica_sets .common - - name: .replica_sets .multi_oplog !.encrypt + - name: .replica_sets .multi_oplog !.encrypt !.requires_large_host + - name: .replica_sets .multi_oplog !.encrypt .requires_large_host + distros: + - rhel8.8-large - name: replica_sets_multi_stmt_txn_jscore_passthrough_gen - name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen distros: @@ -120,7 +123,10 @@ buildvariants: - name: sharded_multi_stmt_txn_jscore_passthrough_gen distros: - rhel8.8-medium - - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host + distros: + - rhel8.8-large - name: .sharding .common !.multiversion !.csrs !.encrypt !.feature_flag_guarded - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen - name: .ssl @@ -171,6 +177,8 @@ buildvariants: - name: multiversion_auth_gen - name: multiversion_gen - name: noPassthrough_gen + distros: + - rhel8.8-large - name: .replica_sets !.multi_oplog !.large - name: .replica_sets !.multi_oplog .large distros: diff --git a/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml index e03ba7bb3fe..421f5cca137 100644 --- a/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml @@ -461,6 +461,8 @@ buildvariants: distros: - rhel8.8-large - name: hotBackups_gen + distros: + - rhel8.8-large - name: generate_buildid_to_debug_symbols_mapping - name: &enterprise-rhel8-sdam-replica-set-monitor-64-bit enterprise-rhel8-sdam-replica-set-monitor-64-bit @@ -487,7 +489,10 @@ buildvariants: - rhel8.8-large - name: .concurrency .common !.kill_terminate !.feature_flag_guarded - name: .jscore .common !.requires_large_host - - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host + distros: + - rhel8.8-large - name: .sharding .common !.csrs !.encrypt !.feature_flag_guarded - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen - name: generate_buildid_to_debug_symbols_mapping @@ -529,6 +534,8 @@ buildvariants: - name: .multi_shard - name: multiversion_auth_gen - name: noPassthrough_gen + distros: + - rhel8.8-large - name: .query_fuzzer - name: .replica_sets !.large - name: .resharding_fuzzer @@ -588,8 +595,12 @@ buildvariants: patch_only: true distros: - rhel8.8-large - - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded !.requires_large_host patch_only: true + - name: .sharding .jscore !.wo_snapshot !.multi_stmt !.feature_flag_guarded .requires_large_host + patch_only: true + distros: + - rhel8.8-large - name: .sharding .common !.csrs !.encrypt !.feature_flag_guarded patch_only: true - name: sharding_max_mirroring_opportunistic_secondary_targeting_gen diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml index 2e7943fc620..2d380825f05 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev.yml @@ -134,7 +134,6 @@ variables: # variant: rhel8-debug-aubsan-compile # THIS HAS COPIES IN: - # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &linux_debug_aubsan_compile_variant_dependency @@ -846,6 +845,8 @@ buildvariants: distros: - rhel8.8-large - name: noPassthrough_gen + distros: + - rhel8.8-large - name: sharding_gen - name: &rhel8-debug-ubsan-roll-back-incremental-feature-flags rhel8-debug-ubsan-roll-back-incremental-feature-flags diff --git a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml index c24ce0fecc4..95f24331f07 100644 --- a/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml @@ -32,24 +32,6 @@ variables: # - name: generate_buildid_to_debug_symbols_mapping # variant: rhel8-debug-aubsan-compile - # THIS HAS COPIES IN: - # - etc/evergreen_yml_components/variants/sanitizer/test_dev_master_branch_only.yml - # - etc/evergreen_yml_components/variants/sanitizer/test_dev.yml - # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - - &linux_debug_aubsan_compile_variant_dependency - depends_on: - - name: archive_dist_test - variant: &linux_debug_aubsan_compile_variant_name linux-debug-aubsan-compile-required - - name: archive_jstestshell - variant: *linux_debug_aubsan_compile_variant_name - - name: version_gen - variant: generate-tasks-for-version - # This is added because of EVG-18211. - # Without this we are adding extra dependencies on evergreen and it is causing strain - omit_generated_tasks: true - # - name: generate_buildid_to_debug_symbols_mapping - # variant: rhel8-debug-aubsan-compile - # THIS HAS COPIES IN: # - etc/evergreen_yml_components/variants/rhel/test_dev_master_branch_only.yml # - etc/evergreen_yml_components/variants/rhel/test_dev_master_and_lts_branches_only.yml diff --git a/evergreen/extract_artifacts.sh b/evergreen/extract_artifacts.sh new file mode 100644 index 00000000000..39dbf5a3dd2 --- /dev/null +++ b/evergreen/extract_artifacts.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -o errexit + +# Extract artifacts from either zstd or gzip compressed tarballs +tar --zstd -xf fetched_artifacts.zst || tar -xf fetched_artifacts.tgz diff --git a/evergreen/write_endor_credentials.sh b/evergreen/write_endor_credentials.sh new file mode 100644 index 00000000000..52da5241b2d --- /dev/null +++ b/evergreen/write_endor_credentials.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Write Endor Labs API credentials to config.yml +# Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, ENDOR_CONFIG_PATH, ENDOR_NAMESPACE + +# use AWS CLI to get the Endor Labs API credentials from AWS Secrets Manager +ENDOR_API_CREDENTIALS_KEY=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_KEY') +ENDOR_API_CREDENTIALS_SECRET=$(aws secretsmanager get-secret-value --secret-id silkbomb-environment --region us-east-1 --query SecretString --output text | jq -r '.ENDOR_API_CREDENTIALS_SECRET') + +# save credentials to config file +mkdir -p ${ENDOR_CONFIG_PATH} +cat <${ENDOR_CONFIG_PATH}/config.yaml +ENDOR_API: https://api.endorlabs.com +ENDOR_API_CREDENTIALS_KEY: $ENDOR_API_CREDENTIALS_KEY +ENDOR_API_CREDENTIALS_SECRET: $ENDOR_API_CREDENTIALS_SECRET +ENDOR_NAMESPACE: ${ENDOR_NAMESPACE} +EOF diff --git a/evergreen/write_silkbomb_env.sh b/evergreen/write_silkbomb_env.sh new file mode 100644 index 00000000000..45aba168911 --- /dev/null +++ b/evergreen/write_silkbomb_env.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Write temporary AWS credentials to Silkbomb environment file +# Requires: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, SILKBOMB_ENV_FILE + +cat <${SILKBOMB_ENV_FILE} +AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} +AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} +AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} +EOF