diff --git a/.gitignore b/.gitignore index 0732b22c45d..b840d6d2513 100644 --- a/.gitignore +++ b/.gitignore @@ -280,6 +280,10 @@ package-lock.json # jstestfuzz generated test directory /jstestfuzz/ +# docker volumes mapped in lint_fuzzer_sanity_patch.py +/jstestfuzzinput +/jstestfuzzoutput + # Bazel-related work # (Note that each string does NOT terminate with a '/' so that gitignore works properly) bazel-* diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 5a87ece939c..b1bead126c6 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1812,14 +1812,9 @@ functions: "setup jstestfuzz": - *f_expansions_write - - command: github.generate_token + - command: ec2.assume_role params: - owner: 10gen - repo: jstestfuzz - expansion_name: github_token - permissions: - metadata: read - contents: read + role_arn: "arn:aws:iam::901841024863:role/ecr-role-evergreen-ro" - command: subprocess.exec display_name: "jstestfuzz setup sh" params: @@ -1827,7 +1822,13 @@ functions: args: - "./src/evergreen/jstestfuzz_setup.sh" include_expansions_in_env: - - github_token + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - command: expansions.update + params: + file: src/expansions.yml + - *f_expansions_write "setup query_tester test repos": - *f_expansions_write @@ -1867,6 +1868,8 @@ functions: binary: bash args: - "src/evergreen/lint_fuzzer_sanity_patch.sh" + include_expansions_in_env: + - CONTAINER_RUNTIME "lint fuzzer sanity all": - *f_expansions_write @@ -1876,6 +1879,8 @@ functions: binary: bash args: - "./src/evergreen/lint_fuzzer_sanity_all.sh" + include_expansions_in_env: + - CONTAINER_RUNTIME "monitor mongo fork 10gen": - *f_expansions_write @@ -1925,6 +1930,8 @@ functions: binary: bash args: - "./src/evergreen/jstestfuzz_run.sh" + include_expansions_in_env: + - CONTAINER_RUNTIME - command: archive.targz_pack display_name: "archive jstests.tgz" params: diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 5cca0f321f6..95cbc1ebba4 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1178,7 +1178,7 @@ tasks: - func: "setup jstestfuzz" - func: "run jstestfuzz" vars: - jstestfuzz_vars: "--numGeneratedFiles 2 --jsTestsDir ../jstests" + jstestfuzz_vars: "--numGeneratedFiles 2 --jsTestsDir jstests" npm_command: "jstestfuzz" - func: "get engflow creds" - func: "bazel test" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index d38184217b9..53fd1a33e77 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -2455,7 +2455,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_causal_consistency resmoke_args: --numClientsPerFixture=10 @@ -2480,7 +2480,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 2 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: --numClientsPerFixture=10 @@ -2509,7 +2509,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 2 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: >- --numClientsPerFixture=10 @@ -2593,7 +2593,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded resmoke_args: --numClientsPerFixture=10 @@ -2616,7 +2616,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_notablescan resmoke_args: --numClientsPerFixture=10 @@ -2638,7 +2638,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_causal_consistency resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2663,7 +2663,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2687,7 +2687,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2711,7 +2711,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests @@ -2734,7 +2734,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_notablescan resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2756,7 +2756,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_sharded_with_config_transitions_and_add_remove_shard resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml index d6eed17eea9..160a002dff2 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml @@ -719,7 +719,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication resmoke_args: --numClientsPerFixture=10 @@ -742,7 +742,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_notablescan resmoke_args: --numClientsPerFixture=10 @@ -764,7 +764,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_write_conflicts resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}' --numClientsPerFixture=10" @@ -789,7 +789,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 8 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_fcbis resmoke_args: --storageEngine=wiredTiger name: jstestfuzz_replication_fcbis @@ -816,7 +816,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -840,7 +840,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 8 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_initsync resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -868,7 +868,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests @@ -891,7 +891,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_notablescan resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -913,7 +913,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication_write_conflicts resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -1841,7 +1841,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_replication resmoke_args: >- --numClientsPerFixture=10 @@ -2054,7 +2054,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz resmoke_args: --numClientsPerFixture=10 @@ -2076,7 +2076,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_interrupt resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2098,7 +2098,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz_interrupt_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml index 69394f0828c..75cacc4cac2 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -433,7 +433,7 @@ tasks: suite: generational_fuzzer use_large_distro: "true" resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - jstestfuzz_vars: --diffTestingMode timeseries --numPipelines=1000 --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 + jstestfuzz_vars: --diffTestingMode timeseries --numPipelines=1000 --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 ## Standalone fuzzer for checking block_processing correctness ## - <<: *jstestfuzz_template @@ -1120,7 +1120,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir ../jstests + jstestfuzz_vars: --jsTestsDir jstests suite: jstestfuzz resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: jstestfuzz @@ -1601,7 +1601,7 @@ tasks: resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests - jstestfuzz_vars: --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 + jstestfuzz_vars: --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 # jstestfuzz standalone update time-series generational fuzzer ## - <<: *jstestfuzz_template @@ -1853,7 +1853,7 @@ tasks: suite: generational_fuzzer use_large_distro: "true" resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - jstestfuzz_vars: --diffTestingMode optimization --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 + jstestfuzz_vars: --diffTestingMode optimization --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 - <<: *task_template name: aggregation_repeat_queries_multiplan_single_solutions diff --git a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml index 6e07158eba5..cf58594b3cc 100644 --- a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml @@ -43,7 +43,8 @@ buildvariants: - name: concurrency_replication_causal_consistency_gen - name: initial_sync_fuzzer_gen - name: .jscore .common !.decimal !.sharding - - name: .jstestfuzz .common !.feature_flag_guarded + # SERVER-105328: Enable this task once container support is added to Windows variants. + # - name: .jstestfuzz .common !.feature_flag_guarded - name: .read_write_concern .linearize !.durable_history - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen diff --git a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml index c39a5142ac3..6de602aa924 100644 --- a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml @@ -170,7 +170,8 @@ buildvariants: distros: - windows-2016-dc - name: .jscore .common !.sharding - - name: .jstestfuzz .common !.feature_flag_guarded + # SERVER-105328: Enable this task once container support is added to Windows variants. + # - name: .jstestfuzz .common !.feature_flag_guarded - name: replica_sets_auth_gen - name: sasl - name: sharding_auth_audit_gen diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh index 1c7e16c1ab3..adb1d4e7bab 100644 --- a/evergreen/jstestfuzz_run.sh +++ b/evergreen/jstestfuzz_run.sh @@ -3,29 +3,16 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" set -evo pipefail -cd src/jstestfuzz - # Store 'jstestfuzz_vars' into 'vars'. We will use 'vars' instead of 'jstestfuzz_vars' for # the rest of this shell script. vars="${jstestfuzz_vars}" -# If the "--jstestfuzzGitRev" option is present in 'vars', copy the option's value into -# 'jstestfuzz_gitrev' and remove it from 'vars', and then reset the jstestfuzz repo to the -# specified git revision. +# # If the "--jstestfuzzGitRev" option is present in 'vars', copy the option's value into +# # 'jstestfuzz_gitrev' and remove it from 'vars'. echo "${vars}" | grep -q -- '--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+' && RET=0 || RET=$? if [ $RET -eq 0 ]; then jstestfuzz_gitrev=$(echo "${vars}" | sed -e 's/.*--jstestfuzzGitRev[ \t]\+\([A-Za-z0-9]\+\).*/\1/') vars=$(echo "${vars}" | sed -e 's/\(.*\)--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+\(.*\)/\1\2/') - - for i in {1..5}; do - git reset --hard "${jstestfuzz_gitrev}" && RET=0 && break || RET=$? && sleep 5 - echo "Failed to reset jstestfuzz to git revision ${jstestfuzz_gitrev}, retrying..." - done - - if [ $RET -ne 0 ]; then - echo "Failed to reset jstestfuzz to git revision ${jstestfuzz_gitrev}" - exit $RET - fi fi # If the "--metaSeed" option is present in 'vars', copy the option's value into 'meta_seed' @@ -73,4 +60,10 @@ if [[ "${npm_command}" != "jstestfuzz" ]]; then maybe_use_es_modules="--useEsModules" fi -./src/scripts/npm_run.sh ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} +mkdir -p ./src/jstestfuzz/out + +$CONTAINER_RUNTIME run --rm \ + -v "./src/jstestfuzz/out:/app/out" \ + -v "./src/jstests:/app/jstests" \ + 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:${jstestfuzz_gitrev:-latest} \ + npm run-script ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index 3c35305b697..fa6e37e2490 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -6,12 +6,18 @@ cd src set -o errexit set -o verbose -for i in {1..5}; do - git clone https://x-access-token:${github_token}@github.com/10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5 - echo "Failed to clone github.com:10gen/jstestfuzz.git, retrying..." -done +if command -v docker > /dev/null 2>&1; then + echo "Docker is installed, using docker" + CONTAINER_RUNTIME=docker -if [ $RET -ne 0 ]; then - echo "Failed to clone git@github.com:10gen/jstestfuzz.git" - exit $RET +elif command -v podman > /dev/null 2>&1; then + echo "Podman is installed, using podman" + CONTAINER_RUNTIME=podman +else + echo "Neither Docker nor Podman is installed. Please install one of them." + exit 1 fi + +echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >> expansions.yml + +aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh index 4a4067585c4..d1701af3c20 100644 --- a/evergreen/lint_fuzzer_sanity_all.sh +++ b/evergreen/lint_fuzzer_sanity_all.sh @@ -1,22 +1,23 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src/jstestfuzz +cd src set -o pipefail set -o verbose -# Run first with help which will do the install -# Then we can run it in parallel -./src/scripts/npm_run.sh --help +LOCAL_INPUT_DIR="$PWD" +CONTAINER_INPUT_DIR="/app" + +# Pull first which will do the install then allow parallel processing below +podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. # Skip javascript files in third_party directory -find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- 2>&1 | tee lint_fuzzer_sanity.log +find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print \ + | xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 \ + | tee lint_fuzzer_sanity.log exit_code=$? -# Exit out of the jstestfuzz directory -cd .. - activate_venv -$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file jstestfuzz/lint_fuzzer_sanity.log --exit-code $exit_code +$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file lint_fuzzer_sanity.log --exit-code $exit_code exit $exit_code diff --git a/evergreen/lint_fuzzer_sanity_patch.py b/evergreen/lint_fuzzer_sanity_patch.py index 9eb74aa39a0..10dfbe76069 100644 --- a/evergreen/lint_fuzzer_sanity_patch.py +++ b/evergreen/lint_fuzzer_sanity_patch.py @@ -40,18 +40,31 @@ for file in diffed_files: os.makedirs(copy_dest.parent, exist_ok=True) shutil.copy(file, copy_dest) -OUTPUT_FULL_DIR = Path(os.getcwd()) / OUTPUT_DIR -INPUT_FULL_DIR = Path(os.getcwd()) / INPUT_DIR +CONTAINER_RUNTIME = os.environ.get("CONTAINER_RUNTIME") +LOCAL_OUTPUT_FULL_DIR = Path(os.getcwd()) / OUTPUT_DIR +LOCAL_INPUT_FULL_DIR = Path(os.getcwd()) / INPUT_DIR + +CONTAINER_INPUT_PATH = f"/app/{INPUT_DIR}" +CONTAINER_OUTPUT_PATH = f"/app/{OUTPUT_DIR}" subprocess.run( [ - "./src/scripts/npm_run.sh", + CONTAINER_RUNTIME, + "run", + "--rm", + "-v", + f"{LOCAL_INPUT_FULL_DIR}:{CONTAINER_INPUT_PATH}", + "-v", + f"{LOCAL_OUTPUT_FULL_DIR}:{CONTAINER_OUTPUT_PATH}", + "901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest", + "npm", + "run-script", "jstestfuzz", "--", "--jsTestsDir", - INPUT_FULL_DIR, + CONTAINER_INPUT_PATH, "--out", - OUTPUT_FULL_DIR, + CONTAINER_OUTPUT_PATH, "--numSourceFiles", str(min(num_changed_files, 100)), "--numGeneratedFiles", @@ -63,7 +76,6 @@ subprocess.run( "10", ], check=True, - cwd="jstestfuzz", ) @@ -73,13 +85,27 @@ def _parse_jsfile(jsfile: Path) -> simple_report.Result: Returns what should be added to the report given to evergreen """ - print(f"Trying to parse jsfile {jsfile}") + # Find the relative path to the jsfile in the volume on the container + relative_js_file_path = f"{INPUT_DIR}/{jsfile.relative_to(LOCAL_INPUT_FULL_DIR)}" start_time = time.time() proc = subprocess.run( - ["./src/scripts/npm_run.sh", "parse-jsfiles", "--", str(jsfile)], + [ + CONTAINER_RUNTIME, + "run", + "--rm", + "-v", + f"{LOCAL_INPUT_FULL_DIR}:{CONTAINER_INPUT_PATH}", + "-v", + f"{LOCAL_OUTPUT_FULL_DIR}:{CONTAINER_OUTPUT_PATH}", + "901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest", + "npm", + "run-script", + "parse-jsfiles", + "--", + str(relative_js_file_path), + ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd="jstestfuzz", ) end_time = time.time() status = "pass" if proc.returncode == 0 else "fail" @@ -104,7 +130,7 @@ report = simple_report.Report(failures=0, results=[]) with futures.ThreadPoolExecutor() as executor: parse_jsfiles_futures = [ executor.submit(_parse_jsfile, Path(jsfile)) - for jsfile in glob.iglob(str(OUTPUT_FULL_DIR / "**"), recursive=True) + for jsfile in glob.iglob(f"{LOCAL_INPUT_FULL_DIR}/**", recursive=True) if os.path.isfile(jsfile) ]