SERVER-76270 Moved lint_fuzzer_sanity to python

buildscripts/evergreen_gen_build_metrics_tasks.py

@@ -134,7 +134,6 @@ def main():
                 FunctionCall("cleanup environment"),
                 FunctionCall("set up venv"),
                 FunctionCall("upload pip requirements"),
-                FunctionCall("get all modified patch files"),
                 FunctionCall("f_expansions_write"),
                 FunctionCall("configure evergreen api credentials"),
                 FunctionCall("get buildnumber"),

buildscripts/linter/filediff.py

@@ -34,7 +34,7 @@ def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]:
     return repos, revision_map
 
 
-def _filter_file(filename: str, is_interesting_file: Callable) -> bool:
+def _filter_file(filename: str, is_interesting_file: Callable[[str], bool]) -> bool:
     """
     Determine if file should be included based on existence and passed in method.
 
@@ -45,7 +45,7 @@ def _filter_file(filename: str, is_interesting_file: Callable) -> bool:
     return os.path.exists(filename) and is_interesting_file(filename)
 
 
-def gather_changed_files_for_lint(is_interesting_file: Callable) -> List[str]:
+def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> List[str]:
     """
     Get the files that have changes since the last git commit.
 

etc/evergreen.yml

@@ -1417,7 +1417,6 @@ buildvariants:
   - name: jsCore_min_batch_repeat_queries_ese_gsm
   - name: jsCore_txns_large_txns_format
   - name: json_schema
-  - name: lint_fuzzer_sanity_patch
   - name: mqlrun
   - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
   - name: multiversion_gen

etc/evergreen_yml_components/definitions.yml

@@ -240,7 +240,6 @@ variables:
   # "set up venv".
   - func: "set up venv"
   - func: "upload pip requirements"
-  - func: "get all modified patch files"
   - func: "f_expansions_write"
   - func: "configure evergreen api credentials"
   - func: "get buildnumber"
@@ -732,23 +731,6 @@ functions:
       args:
       - "./src/evergreen/functions/shared_scons_directory_umount.sh"
 
-  "get all modified patch files":
-  - *f_expansions_write
-  - command: subprocess.exec
-    params:
-      binary: bash
-      args:
-      - "./src/evergreen/functions/modified_patch_files_get_all.sh"
-
-  # This function should only be called from patch-build-only tasks.
-  "get added and modified patch files":
-  - *f_expansions_write
-  - command: subprocess.exec
-    params:
-      binary: bash
-      args:
-      - "./src/evergreen/functions/added_and_modified_patch_files_get.sh"
-
   "determine resmoke jobs": &determine_resmoke_jobs
     command: subprocess.exec
     params:
@@ -1501,7 +1483,8 @@ functions:
     params:
       binary: bash
       args:
-      - "./src/evergreen/lint_fuzzer_sanity_patch.sh"
+      - "src/evergreen/run_python_script.sh"
+      - "evergreen/lint_fuzzer_sanity_patch.py"
 
   "lint fuzzer sanity all":
   - *f_expansions_write
@@ -4520,7 +4503,6 @@ tasks:
   - *kill_processes
   - *cleanup_environment
   - *set_up_venv
-  - func: "get added and modified patch files"
   - func: "setup jstestfuzz"
   - func: "lint fuzzer sanity patch"
 

evergreen/functions/added_and_modified_patch_files_get.sh

@@ -1,14 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/../prelude.sh"
-
-cd src
-
-set -o verbose
-set -o errexit
-
-git diff --name-only origin/${branch_name}... --line-prefix="${workdir}/src/" --diff-filter=d >> modified_and_created_patch_files.txt
-if [ -d src/mongo/db/modules/enterprise ]; then
-  pushd src/mongo/db/modules/enterprise
-  git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >> ~1/modified_and_created_patch_files.txt
-  popd
-fi

evergreen/lint_fuzzer_sanity_patch.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import shutil
+import subprocess
+import glob
+from concurrent import futures
+from pathlib import Path
+import time
+from typing import List, Tuple
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
+
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts import simple_report
+
+# pylint: enable=wrong-import-position
+
+
+def is_js_file(filename: str) -> bool:
+    # return True
+    return filename.startswith("jstests") and filename.endswith(".js")
+
+
+diffed_files = [Path(f) for f in gather_changed_files_for_lint(is_js_file)]
+num_changed_files = len(diffed_files)
+
+if num_changed_files == 0:
+    print("No js files had changes in them. Exiting.")
+    sys.exit(0)
+
+INPUT_DIR = "jstestfuzzinput"
+OUTPUT_DIR = "jstestfuzzoutput"
+os.makedirs(INPUT_DIR, exist_ok=True)
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+for file in diffed_files:
+    copy_dest = INPUT_DIR / file
+    os.makedirs(copy_dest.parent, exist_ok=True)
+    shutil.copy(file, copy_dest)
+
+OUTPUT_FULL_DIR = Path(os.getcwd()) / OUTPUT_DIR
+INPUT_FULL_DIR = Path(os.getcwd()) / INPUT_DIR
+
+subprocess.run([
+    "./src/scripts/npm_run.sh", "jstestfuzz", "--", "--jsTestsDir", INPUT_FULL_DIR, "--out",
+    OUTPUT_FULL_DIR, "--numSourceFiles",
+    str(min(num_changed_files, 250)), "--numGeneratedFiles", "250"
+], check=True, cwd="jstestfuzz")
+
+
+def _parse_jsfile(jsfile: Path) -> simple_report.Result:
+    """
+    Takes in a path to be attempted to parse
+    Returns what should be added to the report given to evergreen
+    """
+    print(f"Trying to parse jsfile {jsfile}")
+    start_time = time.time()
+    proc = subprocess.run(["./src/scripts/npm_run.sh", "parse-jsfiles", "--",
+                           str(jsfile)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                          cwd="jstestfuzz")
+    end_time = time.time()
+    status = "pass" if proc.returncode == 0 else "fail"
+    npm_run_output = proc.stdout.decode("UTF-8")
+    if proc.returncode == 0:
+        print(f"Successfully to parsed jsfile {jsfile}")
+    else:
+        print(f"Failed to parsed jsfile {jsfile}")
+        print(npm_run_output)
+    return simple_report.Result(status=status, exit_code=proc.returncode, start=start_time,
+                                end=end_time, test_file=jsfile.name, log_raw=npm_run_output)
+
+
+report = simple_report.Report(failures=0, results=[])
+
+with futures.ThreadPoolExecutor() as executor:
+    parse_jsfiles_futures = [
+        executor.submit(_parse_jsfile, Path(jsfile))
+        for jsfile in glob.iglob(str(OUTPUT_FULL_DIR / "**"), recursive=True)
+        if os.path.isfile(jsfile)
+    ]
+
+    for future in futures.as_completed(parse_jsfiles_futures):
+        result = future.result()
+        report["results"].append(result)
+        report["failures"] += 1 if result["exit_code"] != 0 else 0
+
+simple_report.put_report(report)
+if report["failures"] > 0:
+    print("Had at least one failure, exiting with 1")
+    sys.exit(1)
+
+print("No failures, exiting success")
+sys.exit(0)

evergreen/lint_fuzzer_sanity_patch.sh

@@ -1,40 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-set -o pipefail
-set -o verbose
-
-activate_venv
-
-mkdir -p jstestfuzzinput jstestfuzzoutput
-
-# We need to be the jstestfuzz repo for node to install/run
-cd jstestfuzz
-
-indir="$(pwd)/../jstestfuzzinput"
-outdir="$(pwd)/../jstestfuzzoutput"
-
-# Grep all the js files from modified_and_created_patch_files.txt and put them into $indir.
-(grep -v "\.tpl\.js$" ../modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true)
-
-# Count the number of files in $indir.
-if [[ "$(ls -A $indir)" ]]; then
-  num_files=$(ls -A $indir | wc -l)
-
-  # Only fetch 50 files to generate jsfuzz testing files.
-  if [[ $num_files -gt 50 ]]; then
-    num_files=50
-  fi
-
-  ./src/scripts/npm_run.sh jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50
-
-  # Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
-  ls -1 -d $outdir/* | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- 2>&1 | tee lint_fuzzer_sanity.log
-  exit_code=$?
-
-  # Exit out of the jstestfuzz directory
-  cd ..
-  $python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_patch --log-file jstestfuzz/lint_fuzzer_sanity.log --exit-code $exit_code
-fi