diff --git a/bazel/auto_header/auto_header.py b/bazel/auto_header/auto_header.py index 469793d80e5..b9e3d6b9e6c 100644 --- a/bazel/auto_header/auto_header.py +++ b/bazel/auto_header/auto_header.py @@ -1,13 +1,11 @@ import subprocess import hashlib import os -import threading import time import json from collections import defaultdict from functools import lru_cache from pathlib import Path -from typing import Dict, List, Set, Tuple from io import StringIO @@ -201,8 +199,8 @@ def _is_third_party_path(p: str) -> bool: ) -def _build_left_rights_map(grouped: Dict[str, Dict[str, Set[str]]]) -> Dict[str, Set[str]]: - out: Dict[str, Set[str]] = {} +def _build_left_rights_map(grouped: dict[str, dict[str, set[str]]]) -> dict[str, set[str]]: + out: dict[str, set[str]] = {} for d, files in grouped.items(): for f, rights in files.items(): out[f"{d}/{f}"] = set(r for r in rights if r.startswith("mongo/")) @@ -221,7 +219,7 @@ def _repo_to_file_label(repo_rel: str) -> str: def _dfs_flatten_from_seed( - seed_repo_path: str, lr_map: Dict[str, Set[str]], visited: Set[str], out_labels: Set[str] + seed_repo_path: str, lr_map: dict[str, set[str]], visited: set[str], out_labels: set[str] ) -> None: srp = seed_repo_path.replace("\\", "/") if _is_third_party_path(srp) or _is_excluded_right(srp): @@ -335,15 +333,15 @@ def _cleanup_from_manifest(prev_dirs: set[str], curr_dirs: set[str]) -> int: return removed -def _compute_flat_idl_from_lr_map(lr_map: Dict[str, Set[str]], seeds: List[str]) -> Set[str]: - out: Set[str] = set() - vis: Set[str] = set() +def _compute_flat_idl_from_lr_map(lr_map: dict[str, set[str]], seeds: list[str]) -> set[str]: + out: set[str] = set() + vis: set[str] = set() for seed in seeds: _dfs_flatten_from_seed(seed, lr_map, vis, out) return {lab for lab in out if "third_party" not in lab.split("/")} -def _inject_flat_group(grouped: Dict[str, Dict[str, Set[str]]], flat_labels: Set[str]) -> None: +def _inject_flat_group(grouped: dict[str, dict[str, set[str]]], flat_labels: set[str]) -> None: # Store as a synthetic "left" whose rights are already absolute labels. # We'll special-case its rendering to dump labels verbatim. grouped.setdefault("mongo", {}).setdefault("idl_headers_flat", set()).update(flat_labels) @@ -366,7 +364,7 @@ def augment_with_idl_placeholders( def augment_with_generated_left( - grouped: Dict[str, Dict[str, Set[str]]], gen_map: Dict[str, List[str]] + grouped: dict[str, dict[str, set[str]]], gen_map: dict[str, list[str]] ) -> None: """ For each generated left entry, ensure a filegroup exists and add the @@ -388,7 +386,7 @@ def augment_with_generated_left( def augment_with_source_placeholders( - grouped: Dict[str, Dict[str, Set[str]]], src_paths: List[str] + grouped: dict[str, dict[str, set[str]]], src_paths: list[str] ) -> None: """Seed a filegroup for every left source/header even if it has 0 includes.""" for p in src_paths: @@ -548,7 +546,7 @@ package(default_visibility = [{vis_list}]) """ -def _dir_digest(src_dir: str, files_map: Dict[str, Set[str]], *, visibility: str) -> str: +def _dir_digest(src_dir: str, files_map: dict[str, set[str]], *, visibility: str) -> str: lines = [] lines.append(f"{VERSION_SALT}|{visibility}|{src_dir}\n") for left in sorted(files_map): @@ -573,7 +571,7 @@ def _read_existing_digest(path: Path) -> str: def _build_content_for_dir( - src_dir: str, file_to_rights: Dict[str, Set[str]], *, visibility: list[str], digest: str + src_dir: str, file_to_rights: dict[str, set[str]], *, visibility: list[str], digest: str ) -> str: pkg = f"{SRC_ROOT_POSIX}/{src_dir}/{AUTO_DIR}" buf = StringIO() @@ -589,8 +587,8 @@ def _build_content_for_dir( write("filegroup(\n") write(f' name = "{emit_name}",\n') - seen: Set[str] = set() - labels: List[str] = [] + seen: set[str] = set() + labels: list[str] = [] if not is_flat: for own in _owning_labels_for_left(src_dir, left_file): @@ -623,7 +621,7 @@ def _build_content_for_dir( return buf.getvalue() -def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: Dict[str, Set[str]]) -> str | None: +def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: dict[str, set[str]]) -> str | None: if not gen_header_repo.endswith("_gen.h"): return None base = gen_header_repo[: -len("_gen.h")] @@ -637,8 +635,8 @@ def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: Dict[str, Set[str]] def _write_build_for_group( - src_dir: str, files_map: Dict[str, Set[str]], *, fsync_write: bool -) -> Tuple[str, bool]: + src_dir: str, files_map: dict[str, set[str]], *, fsync_write: bool +) -> tuple[str, bool]: out_dir = SRC_ROOT / src_dir / AUTO_DIR out_path = out_dir / BUILD_NAME visibility = module_visibility_for_build_dir(src_dir) @@ -736,7 +734,7 @@ def collect_left_right_pairs(rg_bin: str) -> list[tuple[str, str]]: return list(merged) -def parse_left_right_pairs(pairs: list[tuple[str, str]]) -> Dict[str, Dict[str, Set[str]]]: +def parse_left_right_pairs(pairs: list[tuple[str, str]]) -> dict[str, dict[str, set[str]]]: by_srcdir = defaultdict(lambda: defaultdict(set)) add = set.add src_exts = SRC_EXTS_TUPLE @@ -759,7 +757,7 @@ def _env_bool(var: str, default: bool) -> bool: return v not in ("0", "false", "False", "no", "No", "") -def gen_auto_headers(repo_root: Path) -> Dict[str, object]: +def gen_auto_headers(repo_root: Path) -> dict[str, object]: """ Runs the header generation once and returns a result dict: { ok: bool, err: str|None, wrote: int, skipped: int, t_ms: float } diff --git a/bazel/config/generate_config_header.py b/bazel/config/generate_config_header.py index 569b69d59cc..f9ef0e5ed00 100644 --- a/bazel/config/generate_config_header.py +++ b/bazel/config/generate_config_header.py @@ -10,10 +10,9 @@ import inspect import os import sys import textwrap -from typing import Dict -def write_config_header(input_path: str, output_path: str, definitions: Dict[str, str]) -> None: +def write_config_header(input_path: str, output_path: str, definitions: dict[str, str]) -> None: with open(input_path) as in_file: content = in_file.read() diff --git a/bazel/format/rules_lint_format_wrapper.py b/bazel/format/rules_lint_format_wrapper.py index ca4780d7b0f..afeba003162 100644 --- a/bazel/format/rules_lint_format_wrapper.py +++ b/bazel/format/rules_lint_format_wrapper.py @@ -2,7 +2,7 @@ import argparse import os import pathlib import subprocess -from typing import List, Union +from typing import Union from git import Repo from utils.evergreen_git import get_mongodb_remote @@ -45,7 +45,7 @@ def _git_unstaged_files() -> str: return result.stdout.strip() + os.linesep -def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]: +def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> list[str]: """Query git to get a list of files in the repo from a diff.""" # There are 3 diffs we run: # 1. List of commits between origin/master and HEAD of current branch @@ -72,7 +72,7 @@ def run_rules_lint( rules_lint_format_path: pathlib.Path, rules_lint_format_check_path: pathlib.Path, check: bool, - files_to_format: Union[List[str], str] = "all", + files_to_format: Union[list[str], str] = "all", ) -> bool: try: if check: @@ -91,7 +91,7 @@ def run_rules_lint( def run_prettier( - prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all" + prettier: pathlib.Path, check: bool, files_to_format: Union[list[str], str] = "all" ) -> bool: # Explicitly ignore anything in the output directories or any symlinks in the root of the repository # to prevent bad symlinks from failing the run, see https://github.com/prettier/prettier/issues/11568 as @@ -218,7 +218,7 @@ def main() -> int: else: files_to_format = _get_files_changed_since_fork_point(origin_branch) - def files_to_format_contains_bazel_file(files: Union[List[str], str]) -> bool: + def files_to_format_contains_bazel_file(files: Union[list[str], str]) -> bool: if files == "all": return True return any(file.endswith(".bazel") or "BUILD" in file for file in files) diff --git a/bazel/gpg_signer.py b/bazel/gpg_signer.py index a3bd1983636..09be89691ed 100644 --- a/bazel/gpg_signer.py +++ b/bazel/gpg_signer.py @@ -18,7 +18,7 @@ import shutil import subprocess import sys import tempfile -from typing import List, Optional +from typing import Optional debug = False # manually change to enable verbose output @@ -28,7 +28,7 @@ def _debug(msg: str) -> None: print(msg, file=sys.stderr) -def _run(argv: List[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess: +def _run(argv: list[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess: if capture_stdout: return subprocess.run( argv, check=True, text=True, stdout=subprocess.PIPE, stderr=sys.stderr @@ -51,7 +51,7 @@ def _extract_fingerprint(colons_output: str) -> Optional[str]: return None -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: _debug("Starting gpg_signer.py") if len(argv) != 6: @@ -106,7 +106,7 @@ def main(argv: List[str]) -> int: return 1 # Build passphrase options if provided. - pass_opts: List[str] = [] + pass_opts: list[str] = [] if passphrase_file: pass_opts = ["--pinentry-mode", "loopback", "--passphrase-file", passphrase_file] diff --git a/bazel/merge_tidy_configs.py b/bazel/merge_tidy_configs.py index 1e088d9b55c..24fd732dd5c 100644 --- a/bazel/merge_tidy_configs.py +++ b/bazel/merge_tidy_configs.py @@ -13,14 +13,14 @@ from __future__ import annotations import argparse import pathlib -from typing import Any, Dict, List +from typing import Any import yaml # --------------------------- YAML helpers --------------------------- -def load_yaml(file_path: str | pathlib.Path) -> Dict[str, Any]: +def load_yaml(file_path: str | pathlib.Path) -> dict[str, Any]: path = pathlib.Path(file_path) if not path.exists(): raise SystemExit(f"Error: Config file '{file_path}' not found.") @@ -28,11 +28,11 @@ def load_yaml(file_path: str | pathlib.Path) -> Dict[str, Any]: return yaml.safe_load(f) or {} -def split_checks_to_list(value: Any) -> List[str]: +def split_checks_to_list(value: Any) -> list[str]: """Normalize a Checks value (string or list) into a flat list of tokens.""" if value is None: return [] - parts: List[str] = [] + parts: list[str] = [] if isinstance(value, str): parts = [s.strip() for s in value.split(",")] elif isinstance(value, list): @@ -42,7 +42,7 @@ def split_checks_to_list(value: Any) -> List[str]: def merge_checks_into_config( - target_config: Dict[str, Any], incoming_config: Dict[str, Any] + target_config: dict[str, Any], incoming_config: dict[str, Any] ) -> None: """Append incoming Checks onto target Checks (string-concatenated).""" accumulated = split_checks_to_list(target_config.get("Checks")) @@ -51,9 +51,9 @@ def merge_checks_into_config( target_config["Checks"] = ",".join(accumulated + additions) -def check_options_list_to_map(value: Any) -> Dict[str, Any]: +def check_options_list_to_map(value: Any) -> dict[str, Any]: """Transform CheckOptions list[{key,value}] into a dict[key]=value.""" - out: Dict[str, Any] = {} + out: dict[str, Any] = {} if isinstance(value, list): for item in value: if isinstance(item, dict) and "key" in item: @@ -62,7 +62,7 @@ def check_options_list_to_map(value: Any) -> Dict[str, Any]: def merge_check_options_into_config( - target_config: Dict[str, Any], incoming_config: Dict[str, Any] + target_config: dict[str, Any], incoming_config: dict[str, Any] ) -> None: """ Merge CheckOptions so later configs override earlier by 'key'. @@ -161,9 +161,9 @@ def main() -> None: parser.add_argument("--out", required=True, help="Output merged YAML path.") args = parser.parse_args() - merged_config: Dict[str, Any] = load_yaml(args.baseline) + merged_config: dict[str, Any] = load_yaml(args.baseline) - config_paths: List[pathlib.Path] = filter_and_sort_config_paths( + config_paths: list[pathlib.Path] = filter_and_sort_config_paths( args.config_files, args.scope_dir ) diff --git a/bazel/mongot_extension_signing_key/gpg_export_armored_key.py b/bazel/mongot_extension_signing_key/gpg_export_armored_key.py index 05d59277e4b..b5129287798 100644 --- a/bazel/mongot_extension_signing_key/gpg_export_armored_key.py +++ b/bazel/mongot_extension_signing_key/gpg_export_armored_key.py @@ -14,7 +14,7 @@ import shutil import subprocess import sys import tempfile -from typing import List, Optional +from typing import Optional debug = False # manually change to enable verbose output @@ -24,7 +24,7 @@ def _debug(msg: str) -> None: print(msg, file=sys.stderr) -def _run(argv: List[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess: +def _run(argv: list[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess: if capture_stdout: return subprocess.run( argv, check=True, text=True, stdout=subprocess.PIPE, stderr=sys.stderr @@ -47,7 +47,7 @@ def _extract_fingerprint(colons_output: str) -> Optional[str]: return None -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: _debug("Starting gpg_export_armored_key.py") if len(argv) != 5: @@ -104,7 +104,7 @@ def main(argv: List[str]) -> int: _debug("gpg-agent extracted fingerprint: " + fpr) # Build passphrase options if provided. - pass_opts: List[str] = [] + pass_opts: list[str] = [] if passphrase_file: pass_opts = ["--pinentry-mode", "loopback", "--passphrase-file", passphrase_file] diff --git a/bazel/symbol_checker/symbol_checker.py b/bazel/symbol_checker/symbol_checker.py index 441ce76be5b..73ef495c78a 100644 --- a/bazel/symbol_checker/symbol_checker.py +++ b/bazel/symbol_checker/symbol_checker.py @@ -1,7 +1,6 @@ import sys import argparse import json -import os parser = argparse.ArgumentParser() parser.add_argument("--sym", required=True) diff --git a/bazel/wrapper_hook/flag_sync.py b/bazel/wrapper_hook/flag_sync.py index 9141c9c3dff..b1daf9b7b40 100644 --- a/bazel/wrapper_hook/flag_sync.py +++ b/bazel/wrapper_hook/flag_sync.py @@ -3,7 +3,6 @@ import pathlib import sys import time import traceback -from typing import Dict REPO_ROOT = pathlib.Path(__file__).parent.parent.parent sys.path.append(str(REPO_ROOT)) @@ -23,7 +22,7 @@ ALLOW_LINES = [ ] -def update_bazelrc(flags: Dict[str, Dict], verbose: bool): +def update_bazelrc(flags: dict[str, dict], verbose: bool): bazelrc_path = f"{REPO_ROOT}/.bazelrc.sync" if verbose: print(f"Updating {bazelrc_path}") diff --git a/bazel/wrapper_hook/lint.py b/bazel/wrapper_hook/lint.py index 48fa2f08f83..0b00524461b 100644 --- a/bazel/wrapper_hook/lint.py +++ b/bazel/wrapper_hook/lint.py @@ -6,7 +6,7 @@ import shutil import subprocess import sys import tempfile -from typing import List, Optional +from typing import Optional REPO_ROOT = pathlib.Path(__file__).parent.parent.parent sys.path.append(str(REPO_ROOT)) @@ -82,7 +82,7 @@ all_subpackage_javascript_files() print(f"Created BUILD.bazel in {full_dir}") -def list_files_with_targets(bazel_bin: str) -> List: +def list_files_with_targets(bazel_bin: str) -> list: return [ line.strip() for line in subprocess.run( @@ -102,10 +102,10 @@ class LintRunner: def list_files_without_targets( self, - files_with_targets: List[str], + files_with_targets: list[str], type_name: str, ext: str, - dirs: List[str], + dirs: list[str], ) -> bool: # rules_lint only checks files that are in targets, verify that all files in the source tree # are contained within targets. @@ -173,14 +173,14 @@ class LintRunner: else: print(f"All {type_name} files have BUILD.bazel targets!") - def run_bazel(self, target: str, args: List = []): + def run_bazel(self, target: str, args: list = []): p = subprocess.run([self.bazel_bin, "run", target] + (["--"] + args if args else [])) if p.returncode != 0: self.fail = True if not self.keep_going: raise LinterFail("Linter failed") - def simple_file_size_check(self, files_to_lint: List[str]): + def simple_file_size_check(self, files_to_lint: list[str]): for file in files_to_lint: if os.path.getsize(file) > LARGE_FILE_THRESHOLD: print(f"File {file} exceeds large file threshold of {LARGE_FILE_THRESHOLD}") @@ -302,7 +302,7 @@ def _git_unstaged_files() -> str: return result.stdout.strip() + os.linesep -def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]: +def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> list[str]: """Query git to get a list of files in the repo from a diff.""" # There are 3 diffs we run: # 1. List of commits between origin/master and HEAD of current branch @@ -379,7 +379,7 @@ def lint_mod(lint_runner: LintRunner): # subprocess.run([bazel_bin, "run", "//modules_poc:browse", "--", "merged_decls.json", "--parse-only"], check=True) -def run_rules_lint(bazel_bin: str, args: List[str]): +def run_rules_lint(bazel_bin: str, args: list[str]): parsed_args, args = get_parsed_args(args) if platform.system() == "Windows": print("eslint not supported on windows") diff --git a/buildscripts/bazel_burn_in.py b/buildscripts/bazel_burn_in.py index 4842f42d485..e935afceabc 100644 --- a/buildscripts/bazel_burn_in.py +++ b/buildscripts/bazel_burn_in.py @@ -27,7 +27,7 @@ import re import subprocess import sys from functools import cache -from typing import List, NamedTuple +from typing import NamedTuple import typer import yaml @@ -141,7 +141,7 @@ def get_resmoke_configs(): return yaml.safe_load(f) -def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]: +def query_targets_to_burn_in(origin_rev: str) -> list[BurnInTargetInfo]: change_detector = LocalFileChangeDetector(origin_rev) tests_changed = change_detector.find_changed_tests([Repo(".")]) @@ -183,7 +183,7 @@ def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]: @cache -def get_targets_with_tag(tag: str) -> List[str]: +def get_targets_with_tag(tag: str) -> list[str]: try: query = f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', //...)" result = subprocess.run( diff --git a/buildscripts/bazel_custom_formatter.py b/buildscripts/bazel_custom_formatter.py index 2ec5b12c61a..daa0c6dcd7d 100644 --- a/buildscripts/bazel_custom_formatter.py +++ b/buildscripts/bazel_custom_formatter.py @@ -7,7 +7,6 @@ import sys import time from collections import deque from pathlib import Path -from typing import Dict, List sys.path.append(".") @@ -54,7 +53,7 @@ def find_group(unittest_paths): "f": "eighth", } - group_to_path: Dict[str, List[str]] = {} + group_to_path: dict[str, list[str]] = {} for path in unittest_paths: norm_path = path.replace(":", "/").replace("\\", "/") diff --git a/buildscripts/bazel_rules_mongo/codeowners/check_github_codeowner_errors.py b/buildscripts/bazel_rules_mongo/codeowners/check_github_codeowner_errors.py index baf586cd027..c451392dbc0 100644 --- a/buildscripts/bazel_rules_mongo/codeowners/check_github_codeowner_errors.py +++ b/buildscripts/bazel_rules_mongo/codeowners/check_github_codeowner_errors.py @@ -9,7 +9,6 @@ import requests import structlog import yaml -from typing import Dict LOGGER = structlog.get_logger(__name__) @@ -195,7 +194,7 @@ def format_error(error: dict) -> str: return "\n".join(parts) -def get_expansions(expansions_file: str) -> Dict[str, any]: +def get_expansions(expansions_file: str) -> dict[str, any]: if not expansions_file: return None diff --git a/buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py b/buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py index be88710b53c..fbc53a96c14 100644 --- a/buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py +++ b/buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py @@ -5,7 +5,7 @@ import subprocess import sys import tempfile from functools import cache -from typing import Dict, List, Optional, Set, Tuple +from typing import Optional import yaml from codeowners.parsers import owners_v1, owners_v2 @@ -21,12 +21,12 @@ parsers = { class FileNode: def __init__(self, directory: str): - self.dirs: Dict[str, FileNode] = {} + self.dirs: dict[str, FileNode] = {} self.owners_file: Optional[str] = None self.directory = directory -def add_file_to_tree(root_node: FileNode, file_parts: List[str]): +def add_file_to_tree(root_node: FileNode, file_parts: list[str]): current_node = root_node for i, dir in enumerate(file_parts[:-1]): node_dirs = current_node.dirs @@ -42,7 +42,7 @@ def add_file_to_tree(root_node: FileNode, file_parts: List[str]): current_node.owners_file = file_parts[-1] -def build_tree(files: List[str]) -> FileNode: +def build_tree(files: list[str]) -> FileNode: root_node = FileNode("./") for file in files: file_parts = file.split("/") @@ -117,7 +117,7 @@ def validate_generated_codeowners(validator_path: str) -> int: @cache def get_unowned_and_default_owned_files( codeowners_binary_path: str, codeowners_file: str = None -) -> Tuple[Set[str], Set[str]]: +) -> tuple[set[str], set[str]]: temp_output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".txt") temp_output_file.close() default_owner = get_default_owner() @@ -327,7 +327,7 @@ def get_default_owner() -> Optional[str]: @cache -def get_allowed_unowned_files() -> Set[str]: +def get_allowed_unowned_files() -> set[str]: allowed_unowned_file_path = get_allowed_unowned_files_path() if not allowed_unowned_file_path: return set() @@ -367,7 +367,7 @@ def get_allowed_unowned_files() -> Set[str]: return unowned_files -def add_allowed_unowned_files(output_lines: List[str]) -> None: +def add_allowed_unowned_files(output_lines: list[str]) -> None: allowed_unowned_files = get_allowed_unowned_files() if not allowed_unowned_files: return diff --git a/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py b/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py index 8c98e18ebdf..1f3e6ef3c54 100644 --- a/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py +++ b/buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py @@ -2,14 +2,13 @@ import glob import os import pathlib from functools import cache -from typing import Dict, List import yaml # Parser for OWNERS.yml files version 1.0.0 class OwnersParserV1: - def parse(self, directory: str, owners_file_path: str, contents: Dict[str, any]) -> List[str]: + def parse(self, directory: str, owners_file_path: str, contents: dict[str, any]) -> list[str]: lines = [] no_parent_owners = False if "options" in contents: @@ -71,7 +70,7 @@ class OwnersParserV1: return lines @cache - def process_alias_import(self, path: str) -> Dict[str, List[str]]: + def process_alias_import(self, path: str) -> dict[str, list[str]]: if not path.startswith("//"): raise RuntimeError( f"Alias file paths must start with // and be relative to the repo root: {path}" diff --git a/buildscripts/bazel_rules_mongo/engflow_auth/engflow_auth.py b/buildscripts/bazel_rules_mongo/engflow_auth/engflow_auth.py index 84df6592188..77672d899fe 100644 --- a/buildscripts/bazel_rules_mongo/engflow_auth/engflow_auth.py +++ b/buildscripts/bazel_rules_mongo/engflow_auth/engflow_auth.py @@ -2,7 +2,6 @@ Install engflow_auth binary if not on system and authenticate if token is expired. """ -import argparse import hashlib import json import os diff --git a/buildscripts/bazel_rules_mongo/utils/evergreen_git.py b/buildscripts/bazel_rules_mongo/utils/evergreen_git.py index 602554bc83b..1340283cec6 100644 --- a/buildscripts/bazel_rules_mongo/utils/evergreen_git.py +++ b/buildscripts/bazel_rules_mongo/utils/evergreen_git.py @@ -1,13 +1,13 @@ import os from functools import cache -from typing import Dict, List, Optional +from typing import Optional import yaml from git import Remote, Repo @cache -def get_expansions(expansions_file: str) -> Dict[str, any]: +def get_expansions(expansions_file: str) -> dict[str, any]: if not expansions_file: return None @@ -89,7 +89,7 @@ def get_remote_branch_ref(repo: Repo, branch: str = None) -> str: return repo.git.merge_base(local_head.hexsha, remote_head.hexsha) -def get_new_files(expansions_file: str = None, branch: str = None) -> List[str]: +def get_new_files(expansions_file: str = None, branch: str = None) -> list[str]: # docs on the diff-filter are here https://www.kernel.org/pub/software/scm/git/docs/git-diff.html # This gets added, renamed, and copied files from the git diff. return get_changed_files(expansions_file, branch, diff_filter="ARC") @@ -126,7 +126,7 @@ def get_diff_revision(expansions_file: str = None, branch: str = None) -> str: def get_changed_files( expansions_file: str = None, branch: str = None, diff_filter: str | None = "d" -) -> List[str]: +) -> list[str]: diff_commit = get_diff_revision(expansions_file, branch) repo = Repo() @@ -151,7 +151,7 @@ def get_file_at_revision(file: str, revision: str) -> Optional[str]: raise ex -def get_files_to_lint() -> List[str]: +def get_files_to_lint() -> list[str]: # Returns all tracked files and unstaged files repo = Repo() # all tracked files by git diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index c3c160bffc1..704b38b8335 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -11,7 +11,7 @@ import subprocess import sys from abc import ABC, abstractmethod from collections import defaultdict -from typing import Dict, List, NamedTuple, Optional, Set, Tuple +from typing import NamedTuple, Optional import click import structlog @@ -158,7 +158,7 @@ def is_file_a_test_file(file_path: str) -> bool: return True -def find_excludes(selector_file: str) -> Tuple[List, List, List]: +def find_excludes(selector_file: str) -> tuple[list, list, list]: """Parse etc/burn_in_tests.yml. Returns lists of excluded suites, tasks & tests.""" if not selector_file: @@ -180,7 +180,7 @@ def find_excludes(selector_file: str) -> Tuple[List, List, List]: ) -def filter_tests(tests: Set[str], exclude_tests: List[str]) -> Set[str]: +def filter_tests(tests: set[str], exclude_tests: list[str]) -> set[str]: """ Exclude tests which have been denylisted. @@ -251,7 +251,7 @@ class SuiteToBurnInInfo(NamedTuple): name: str resmoke_args: str - tests: List[str] + tests: list[str] class TaskToBurnInInfo(NamedTuple): @@ -263,13 +263,13 @@ class TaskToBurnInInfo(NamedTuple): """ display_task_name: str - suites: List[SuiteToBurnInInfo] + suites: list[SuiteToBurnInInfo] @classmethod def from_task( cls, task: VariantTask, - tests_by_suite: Dict[str, List[str]], + tests_by_suite: dict[str, list[str]], ) -> "TaskToBurnInInfo": """ Gather the information needed to run the given task. @@ -296,9 +296,9 @@ class TaskToBurnInInfo(NamedTuple): def create_task_list( evergreen_conf: EvergreenProjectConfig, build_variant: str, - tests_by_suite: Dict[str, List[str]], + tests_by_suite: dict[str, list[str]], exclude_tasks: [str], -) -> Dict[str, TaskToBurnInInfo]: +) -> dict[str, TaskToBurnInInfo]: """ Find associated tasks for the specified build_variant and suites. @@ -334,8 +334,8 @@ def create_task_list( def _process_tests_by_suite( - task: VariantTask, tests_by_suite: Dict[str, List[str]] -) -> Dict[str, List[str]]: + task: VariantTask, tests_by_suite: dict[str, list[str]] +) -> dict[str, list[str]]: """Filter tests that should run under task according to build variant and task configuration.""" suite_to_run_options_map = task.combined_suite_to_resmoke_args_map tests_by_suite_for_task = defaultdict(list) @@ -369,12 +369,12 @@ def _set_resmoke_cmd(repeat_config: RepeatConfig, resmoke_args: [str]) -> [str]: def create_task_list_for_tests( - changed_tests: Set[str], + changed_tests: set[str], build_variant: str, evg_conf: EvergreenProjectConfig, - exclude_suites: Optional[List] = None, - exclude_tasks: Optional[List] = None, -) -> Dict[str, TaskToBurnInInfo]: + exclude_suites: Optional[list] = None, + exclude_tasks: Optional[list] = None, +) -> dict[str, TaskToBurnInInfo]: """ Create a list of tests by task for the given tests. @@ -402,8 +402,8 @@ def create_task_list_for_tests( def create_tests_by_task( build_variant: str, evg_conf: EvergreenProjectConfig, - changed_tests: Set[str], -) -> Dict[str, TaskToBurnInInfo]: + changed_tests: set[str], +) -> dict[str, TaskToBurnInInfo]: """ Create a list of tests by task. @@ -429,7 +429,7 @@ def create_tests_by_task( return {} -def run_tests(tests_by_task: Dict[str, TaskToBurnInInfo], resmoke_cmd: [str]) -> None: +def run_tests(tests_by_task: dict[str, TaskToBurnInInfo], resmoke_cmd: [str]) -> None: """ Run the given tests locally. @@ -488,7 +488,7 @@ class FileChangeDetector(ABC): """Interface to detect changes to files.""" @abstractmethod - def create_revision_map(self, repos: List[Repo]) -> RevisionMap: + def create_revision_map(self, repos: list[Repo]) -> RevisionMap: """ Create a map of the repos and the given revisions to diff against. @@ -497,7 +497,7 @@ class FileChangeDetector(ABC): """ raise NotImplementedError() - def find_changed_tests(self, repos: List[Repo]) -> Set[str]: + def find_changed_tests(self, repos: list[Repo]) -> set[str]: """ Find the changed tests. @@ -525,7 +525,7 @@ class LocalFileChangeDetector(FileChangeDetector): """ self.origin_rev = origin_rev - def create_revision_map(self, repos: List[Repo]) -> RevisionMap: + def create_revision_map(self, repos: list[Repo]) -> RevisionMap: """ Create a map of the repos and the given revisions to diff against. @@ -542,7 +542,7 @@ class BurnInExecutor(ABC): """An interface to execute discovered tests.""" @abstractmethod - def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: + def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -554,7 +554,7 @@ class BurnInExecutor(ABC): class NopBurnInExecutor(BurnInExecutor): """A burn-in executor that displays results, but doesn't execute.""" - def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: + def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -582,7 +582,7 @@ class LocalBurnInExecutor(BurnInExecutor): self.resmoke_args = resmoke_args self.repeat_config = repeat_config - def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: + def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -602,7 +602,7 @@ class DiscoveredSuite(BaseModel): """ suite_name: str - test_list: List[str] + test_list: list[str] class DiscoveredTask(BaseModel): @@ -614,19 +614,19 @@ class DiscoveredTask(BaseModel): """ task_name: str - suites: List[DiscoveredSuite] + suites: list[DiscoveredSuite] class DiscoveredTaskList(BaseModel): """Model for a list of discovered tasks.""" - discovered_tasks: List[DiscoveredTask] + discovered_tasks: list[DiscoveredTask] class YamlBurnInExecutor(BurnInExecutor): """A burn-in executor that outputs discovered tasks as YAML.""" - def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: + def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None: """ Report the given tasks and their tests to stdout. @@ -667,7 +667,7 @@ class BurnInOrchestrator: self.burn_in_executor = burn_in_executor self.evg_conf = evg_conf - def burn_in(self, repos: List[Repo], build_variant: str) -> None: + def burn_in(self, repos: list[Repo], build_variant: str) -> None: """ Execute burn in tests for the given git repositories. diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 30b9ff60296..2fd510ed149 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -12,7 +12,7 @@ import re import shutil import subprocess import sys -from typing import Any, Dict, List, Optional, Set +from typing import Any, Optional import structlog @@ -113,7 +113,7 @@ class EvergreenProjectConfig(object): self.functions = self._conf["functions"] @property - def task_names(self) -> List[str]: + def task_names(self) -> list[str]: """Get the list of task names.""" return list(self._tasks_by_name.keys()) @@ -139,7 +139,7 @@ class EvergreenProjectConfig(object): """Return the variant with the given name as a Variant instance.""" return self._variants_by_name.get(variant_name) - def get_required_variants(self) -> Set[Variant]: + def get_required_variants(self) -> set[Variant]: """Get the list of required build variants.""" return {variant for variant in self.variants if variant.is_required_variant()} @@ -176,7 +176,7 @@ class Task(object): return None @property - def generate_resmoke_tasks_command(self) -> Optional[Dict[str, Any]]: + def generate_resmoke_tasks_command(self) -> Optional[dict[str, Any]]: """Return the 'generate resmoke tasks' command if found, or None.""" return self.find_func_command("generate resmoke tasks") @@ -186,7 +186,7 @@ class Task(object): return self.generate_resmoke_tasks_command is not None @property - def run_tests_command(self) -> Optional[Dict[str, Any]]: + def run_tests_command(self) -> Optional[dict[str, Any]]: """Return the 'run tests' command if found, or None.""" return self.find_func_command("run tests") @@ -196,7 +196,7 @@ class Task(object): return self.run_tests_command is not None @property - def initialize_multiversion_tasks_command(self) -> Optional[Dict[str, Any]]: + def initialize_multiversion_tasks_command(self) -> Optional[dict[str, Any]]: """Return the 'initialize multiversion tasks' command if found, or None.""" return self.find_func_command("initialize multiversion tasks") @@ -217,7 +217,7 @@ class Task(object): return self.name[:-4] - def get_resmoke_command_vars(self) -> Dict[str, Any]: + def get_resmoke_command_vars(self) -> dict[str, Any]: """Get the vars for either 'generate resmoke tasks' or 'run tests', both eventually call resmoke.py.""" if self.is_run_tests_task: return self.run_tests_command.get("vars", {}) @@ -226,7 +226,7 @@ class Task(object): return {} - def get_suite_names(self) -> List[str]: + def get_suite_names(self) -> list[str]: """Get the names of the resmoke.py suites from the task definition.""" command_vars = self.get_resmoke_command_vars() @@ -243,7 +243,7 @@ class Task(object): raise ValueError(f"{self.name} task does not run a resmoke.py test suite") @property - def suite_to_resmoke_args_map(self) -> Dict[str, str]: + def suite_to_resmoke_args_map(self) -> dict[str, str]: """Get the resmoke.py arguments from the task definition.""" output = {} @@ -439,7 +439,7 @@ class VariantTask(Task): return f"{self.variant}: {self.name}" @property - def combined_suite_to_resmoke_args_map(self) -> Dict[str, str]: + def combined_suite_to_resmoke_args_map(self) -> dict[str, str]: """Get the combined resmoke arguments. This results from the concatenation of the task's resmoke_args parameter and the diff --git a/buildscripts/clang_tidy.py b/buildscripts/clang_tidy.py index ecbc6ffdf3d..f515d65405e 100755 --- a/buildscripts/clang_tidy.py +++ b/buildscripts/clang_tidy.py @@ -15,7 +15,7 @@ import sys import time from concurrent import futures from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional import yaml @@ -41,12 +41,12 @@ for config in ["/tmp/compiledb-bin/.clang-tidy.strict", "bazel-bin/.clang-tidy.s def _clang_tidy_executor( clang_tidy_filename: Path, clang_tidy_binary: str, - clang_tidy_cfg: Dict[str, Any], + clang_tidy_cfg: dict[str, Any], output_dir: str, show_stdout: bool, mongo_check_module: str = "", compile_commands: str = "compile_commands.json", -) -> Tuple[str, Optional[str]]: +) -> tuple[str, Optional[str]]: clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent os.makedirs(clang_tidy_parent_dir, exist_ok=True) @@ -92,7 +92,7 @@ def _clang_tidy_executor( return proc.stdout.decode(locale.getpreferredencoding()), files_to_parse -def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int: +def _combine_errors(fixes_filename: str, files_to_parse: list[str]) -> int: failed_files = 0 all_fixes = {} @@ -140,7 +140,7 @@ def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int: return failed_files -def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str: +def __dedup_errors(clang_tidy_errors_threads: list[str]) -> str: unique_single_errors = set() for errs in clang_tidy_errors_threads: if errs: @@ -211,7 +211,7 @@ def _run_tidy(args, parser_defaults): ) sys.exit(1) - files_to_tidy: List[Path] = list() + files_to_tidy: list[Path] = list() files_to_parse = list() filtered_compile_commands = [] for file_doc in compile_commands: @@ -259,8 +259,8 @@ def _run_tidy(args, parser_defaults): total_jobs = len(files_to_tidy) workers = args.threads - clang_tidy_errors_futures: List[str] = [] - clang_tidy_executor_futures: List[futures.ThreadPoolExecutor.submit] = [] + clang_tidy_errors_futures: list[str] = [] + clang_tidy_executor_futures: list[futures.ThreadPoolExecutor.submit] = [] # total completed tasks tasks_completed = 0 diff --git a/buildscripts/client/jiraclient.py b/buildscripts/client/jiraclient.py index ebde5b1919b..dcc5c178d2d 100644 --- a/buildscripts/client/jiraclient.py +++ b/buildscripts/client/jiraclient.py @@ -2,7 +2,7 @@ import logging from enum import Enum -from typing import Any, Dict, Iterable, Optional, Sequence +from typing import Any, Iterable, Optional, Sequence from jira import JIRA, Issue from jira.client import ResultList @@ -37,7 +37,7 @@ class JiraAuth(BaseSettings): def get_token_auth(self) -> Optional[str]: return self.pat - def get_oauth(self) -> Optional[Dict[str, Any]]: + def get_oauth(self) -> Optional[dict[str, Any]]: if self.access_token and self.access_token_secret and self.consumer_key and self.key_cert: return { "access_token": self.access_token, diff --git a/buildscripts/compare_evergreen_versions.py b/buildscripts/compare_evergreen_versions.py index 9b7777cce35..f25a2866f1d 100755 --- a/buildscripts/compare_evergreen_versions.py +++ b/buildscripts/compare_evergreen_versions.py @@ -37,7 +37,7 @@ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Optional +from typing import Optional # Add the repository root to sys.path to allow imports from buildscripts _SCRIPT_DIR = Path(__file__).resolve().parent @@ -129,7 +129,7 @@ def _fetch_build_tasks( build_variant_name: str, build_id: str, non_running_statuses: set[str], -) -> List[TaskStats]: +) -> list[TaskStats]: """ Fetch tasks for a single build variant. @@ -137,7 +137,7 @@ def _fetch_build_tasks( """ try: build = evg_api.build_by_id(build_id) - tasks: List[Task] = build.get_tasks() + tasks: list[Task] = build.get_tasks() task_stats_list = [] for task in tasks: @@ -172,7 +172,7 @@ def _fetch_build_tasks( def get_version_tasks( evg_api: RetryingEvergreenApi, version_id: str, max_workers: int = 10 -) -> Dict[TaskKey, TaskStats]: +) -> dict[TaskKey, TaskStats]: """ Fetch all tasks for a given version using parallel requests. @@ -297,7 +297,7 @@ def compare_versions( version_id_2: str, task_filter: Optional[str] = None, max_workers: int = 10, -) -> List[TaskComparison]: +) -> list[TaskComparison]: """ Compare tasks between two versions. @@ -376,7 +376,7 @@ def truncate_variant_name(variant_name: str) -> str: def print_table_output( - comparisons: List[TaskComparison], + comparisons: list[TaskComparison], version_id_1: str, version_id_2: str, min_diff_percent: float = 0.0, @@ -560,7 +560,7 @@ def print_table_output( def print_json_output( - comparisons: List[TaskComparison], + comparisons: list[TaskComparison], version_id_1: str, version_id_2: str, ): @@ -601,7 +601,7 @@ def print_json_output( def print_csv_output( - comparisons: List[TaskComparison], + comparisons: list[TaskComparison], version_id_1: str, version_id_2: str, ): diff --git a/buildscripts/create_bazel_test_report.py b/buildscripts/create_bazel_test_report.py index 9cc57107a23..7d31d324318 100644 --- a/buildscripts/create_bazel_test_report.py +++ b/buildscripts/create_bazel_test_report.py @@ -2,7 +2,6 @@ import json import time from glob import glob from pathlib import Path -from typing import List import typer from typing_extensions import TypedDict @@ -24,7 +23,7 @@ class Result(TypedDict): class Report(TypedDict): """Test report for Evergreen""" - results: List[Result] + results: list[Result] def main(testlog_dir: str): diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py index a4832d4beb9..4e52f69a3bf 100644 --- a/buildscripts/debugsymb_mapper.py +++ b/buildscripts/debugsymb_mapper.py @@ -11,7 +11,7 @@ import subprocess import sys import time from json import JSONDecoder -from typing import Dict, Generator, List, NamedTuple, Optional +from typing import Generator, NamedTuple, Optional import requests @@ -33,7 +33,7 @@ class CmdClient: """Client to run commands.""" @staticmethod - def run(args: List[str]) -> str: + def run(args: list[str]) -> str: """ Run command with args. @@ -345,7 +345,7 @@ class Mapper: tarball_full_path = download.download_from_s3(url) return tarball_full_path - def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]: + def generate_build_id_mapping(self) -> Generator[dict[str, str], None, None]: """ Extract build id from binaries and creates new dict using them. diff --git a/buildscripts/evergreen_gen_powercycle_tasks.py b/buildscripts/evergreen_gen_powercycle_tasks.py index 2d5c15a263d..aea8fd167a1 100755 --- a/buildscripts/evergreen_gen_powercycle_tasks.py +++ b/buildscripts/evergreen_gen_powercycle_tasks.py @@ -2,7 +2,7 @@ """Generate multiple powercycle tasks to run in evergreen.""" from collections import namedtuple -from typing import Any, List, Set, Tuple +from typing import Any import click from shrub.v2 import BuildVariant, ExistingTask, FunctionCall, ShrubProject, Task, TaskDependency @@ -68,14 +68,14 @@ def make_config(expansions_file: Any) -> Config: ) -def get_setup_commands() -> Tuple[List[FunctionCall], Set[TaskDependency]]: +def get_setup_commands() -> tuple[list[FunctionCall], set[TaskDependency]]: """Return setup commands.""" return [ FunctionCall("do setup"), ], {TaskDependency("archive_dist_test")} -def get_skip_compile_setup_commands() -> Tuple[List[FunctionCall], set]: +def get_skip_compile_setup_commands() -> tuple[list[FunctionCall], set]: """Return skip compile setup commands.""" return [ BuiltInCommand("manifest.load", {}), diff --git a/buildscripts/evergreen_task_timeout.py b/buildscripts/evergreen_task_timeout.py index c09f4e68384..d561309cf82 100755 --- a/buildscripts/evergreen_task_timeout.py +++ b/buildscripts/evergreen_task_timeout.py @@ -11,7 +11,7 @@ import shlex import sys from datetime import timedelta from pathlib import Path -from typing import Dict, List, Optional +from typing import Optional import inject import structlog @@ -95,7 +95,7 @@ class TimeoutOverride(BaseModel): class TimeoutOverrides(BaseModel): """Collection of timeout overrides to apply.""" - overrides: Dict[str, List[TimeoutOverride]] + overrides: dict[str, list[TimeoutOverride]] @classmethod def from_yaml_file(cls, file_path: Path) -> "TimeoutOverrides": diff --git a/buildscripts/fast_archive.py b/buildscripts/fast_archive.py index 87b2386289a..85a996f6dad 100644 --- a/buildscripts/fast_archive.py +++ b/buildscripts/fast_archive.py @@ -8,7 +8,7 @@ import shutil import subprocess import sys import time -from typing import Dict, List, Optional +from typing import Optional import boto3 import requests @@ -28,7 +28,7 @@ def process_file( file_number: int, upload_name: str, start_time: int, -) -> Optional[Dict[str, str]]: +) -> Optional[dict[str, str]]: print(f"{file} started compressing at {time.time() - start_time}") compressed_file = f"{file}.gz" with open(file, "rb") as f_in: @@ -73,7 +73,7 @@ def process_file( return task_artifact -def main(output_file: str, patterns: List[str], display_name: str, expansions_file: str) -> int: +def main(output_file: str, patterns: list[str], display_name: str, expansions_file: str) -> int: if not output_file.endswith(".json"): print("WARN: filename input should end with `.json`", file=sys.stderr) diff --git a/buildscripts/fix_headers.py b/buildscripts/fix_headers.py index a4395f2fbe3..21fcd44b4dc 100644 --- a/buildscripts/fix_headers.py +++ b/buildscripts/fix_headers.py @@ -12,7 +12,7 @@ import os import pprint import subprocess import sys -from typing import Annotated, Dict, List, Optional, Tuple +from typing import Annotated, Optional import typer @@ -72,7 +72,7 @@ def todo_comment(issue_key: str, header: str, new_dep: str, fg_label: str) -> No bd_utils.bd_comment([fg_label], comment) -def useful_print(fixes: Dict) -> None: +def useful_print(fixes: dict) -> None: for target, target_fixes in fixes.items(): print("-", target) print(" Fixes:\n") @@ -92,7 +92,7 @@ class HeaderFixer: self.team_issues = {} def _query( - self, query: str, config: bool = False, args: List[str] = [] + self, query: str, config: bool = False, args: list[str] = [] ) -> subprocess.CompletedProcess: query_cmd = "cquery" @@ -118,7 +118,7 @@ class HeaderFixer: def _create_header_target(self): pass - def _find_misplaced_headers(self, target: str) -> List[str]: + def _find_misplaced_headers(self, target: str) -> list[str]: p = self._query(f"labels(hdrs,{target}{CC_LIB_SUFFIX})") misplaced_headers = [] target_package = target.split(":")[0] + ":" @@ -134,7 +134,7 @@ class HeaderFixer: return misplaced_headers - def _find_header_target(self, header: str) -> Tuple[Optional[str], bool]: + def _find_header_target(self, header: str) -> tuple[Optional[str], bool]: potential_target = header.split(".")[0] p = self._query(f"attr(srcs,{potential_target}.cpp,//...)") target = None @@ -170,7 +170,7 @@ class HeaderFixer: return False - def _fix_target(self, target: str) -> Dict: + def _fix_target(self, target: str) -> dict: target_fixes = {"fixes": {}, "cycles": {}} orphaned_headers = [] for hdr in self._find_misplaced_headers(target): @@ -208,7 +208,7 @@ class HeaderFixer: print("\n".join(orphaned_headers)) return target_fixes - def _evaluate_target_expression(self, target_exp: str) -> List[str]: + def _evaluate_target_expression(self, target_exp: str) -> list[str]: p = self._query( f"filter('.*{CC_LIB_SUFFIX}$',kind(cc_library,deps({target_exp}, 1)))", ["--noimplicit_deps"], @@ -248,7 +248,7 @@ class HeaderFixer: return "" return issue.key - def fix_targets(self, target_exp: str) -> Dict: + def fix_targets(self, target_exp: str) -> dict: fixes = {} for target in self._evaluate_target_expression(target_exp): fixes[target] = self._fix_target(target) diff --git a/buildscripts/gather_failed_tests.py b/buildscripts/gather_failed_tests.py index d78e06c3f75..f1c87f5fd0b 100644 --- a/buildscripts/gather_failed_tests.py +++ b/buildscripts/gather_failed_tests.py @@ -3,7 +3,6 @@ import os import shutil import subprocess from pathlib import Path -from typing import List import typer @@ -27,7 +26,7 @@ def process_bep(bep_path): return failed_tests, successful_tests -def _relink_binaries_with_symbols(failed_test_labels: List[str]): +def _relink_binaries_with_symbols(failed_test_labels: list[str]): print("Rebuilding tests with --remote_download_outputs=toplevel...") bazel_build_flags = "" if os.path.isfile(".bazel_build_flags"): diff --git a/buildscripts/generate_result_tasks.py b/buildscripts/generate_result_tasks.py index c80d442020a..63fd8377984 100644 --- a/buildscripts/generate_result_tasks.py +++ b/buildscripts/generate_result_tasks.py @@ -20,7 +20,7 @@ import os import subprocess import sys from functools import cache -from typing import List, Optional +from typing import Optional import runfiles import typer @@ -131,7 +131,7 @@ def resolve_codeowners() -> dict[str, list[str]]: return {} -def query_targets() -> List[str]: +def query_targets() -> list[str]: try: result = subprocess.run( ["bazel", "query", RESMOKE_TEST_QUERY], diff --git a/buildscripts/get_agg_stages_no_authz.py b/buildscripts/get_agg_stages_no_authz.py index 92f72cc1bbf..84e02e06a1f 100644 --- a/buildscripts/get_agg_stages_no_authz.py +++ b/buildscripts/get_agg_stages_no_authz.py @@ -1,6 +1,6 @@ import os import re -from typing import Optional, Set +from typing import Optional MONGO_DOCUMENT_SOURCE_DIR = "src/mongo/db/pipeline" @@ -62,7 +62,7 @@ def check_file_pair(base_path: str) -> Optional[str]: return None -def find_authz_opt_out_stages() -> Set[str]: +def find_authz_opt_out_stages() -> set[str]: """ Scans the pipeline directory to find aggregation stages that opted out of authorization checks. @@ -74,8 +74,8 @@ def find_authz_opt_out_stages() -> Set[str]: return set() print(f"Scanning for aggregation stages in '{MONGO_DOCUMENT_SOURCE_DIR}'") - found_stages: Set[str] = set() - processed_bases: Set[str] = set() + found_stages: set[str] = set() + processed_bases: set[str] = set() # Walk through directory for root, _, files in os.walk(MONGO_DOCUMENT_SOURCE_DIR): diff --git a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py index 9f63cf3d05c..a8b34816e78 100644 --- a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py +++ b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py @@ -35,7 +35,6 @@ import logging import os import sys from tempfile import TemporaryDirectory -from typing import Dict, List, Set from pymongo import MongoClient @@ -65,8 +64,8 @@ def is_test_or_third_party_idl(idl_path: str) -> bool: def get_command_definitions( - api_version: str, directory: str, import_directories: List[str] -) -> Dict[str, syntax.Command]: + api_version: str, directory: str, import_directories: list[str] +) -> dict[str, syntax.Command]: """Get parsed IDL definitions of commands in a given API version.""" LOGGER.info("Searching for command definitions in %s", directory) @@ -83,7 +82,7 @@ def get_command_definitions( return idl_commands -def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> Set[str]: +def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> set[str]: """Get a list of commands in a given API version by calling listCommands.""" assert mongod_or_mongos in ("mongod", "mongos") logging.info("Calling listCommands on %s", mongod_or_mongos) @@ -135,7 +134,7 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: fixture.teardown() -def assert_command_sets_equal(api_version: str, command_sets: Dict[str, Set[str]]): +def assert_command_sets_equal(api_version: str, command_sets: dict[str, set[str]]): """Check that all sources have the same set of commands for a given API version.""" LOGGER.info("Comparing %s command sets", len(command_sets)) for name, commands in command_sets.items(): @@ -161,7 +160,7 @@ def assert_command_sets_equal(api_version: str, command_sets: Dict[str, Set[str] ) -def remove_skipped_commands(command_sets: Dict[str, Set[str]]): +def remove_skipped_commands(command_sets: dict[str, set[str]]): """Remove skipped commands from command_sets.""" skipped_commands = { "testDeprecation", diff --git a/buildscripts/idl/checkout_idl_files_from_past_releases.py b/buildscripts/idl/checkout_idl_files_from_past_releases.py index ae377484589..cf4bd68f7fa 100755 --- a/buildscripts/idl/checkout_idl_files_from_past_releases.py +++ b/buildscripts/idl/checkout_idl_files_from_past_releases.py @@ -32,7 +32,6 @@ import os import shutil import sys from subprocess import CalledProcessError, check_output -from typing import List from packaging.version import Version from retry import retry @@ -51,7 +50,7 @@ LOGGER_NAME = "checkout-idl" LOGGER = logging.getLogger(LOGGER_NAME) -def get_tags() -> List[str]: +def get_tags() -> list[str]: """Get a list of git tags that the IDL compatibility script should check against.""" def gen_versions_and_tags(): @@ -111,11 +110,11 @@ def _show_with_retry(tag: str, path: str): @retry(tries=3, delay=5) -def _fetch_with_retry(tags: List[str]): +def _fetch_with_retry(tags: list[str]): return check_output(["git", "fetch", "origin", *tags]) -def make_idl_directories(tags: List[str], destination: str) -> None: +def make_idl_directories(tags: list[str], destination: str) -> None: """For each tag, construct a source tree containing only its IDL files.""" LOGGER.info("Clearing destination directory '%s'", destination) shutil.rmtree(destination, ignore_errors=True) diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py index ea73a2d158a..7d82dfc88bd 100755 --- a/buildscripts/idl/gen_all_feature_flag_list.py +++ b/buildscripts/idl/gen_all_feature_flag_list.py @@ -32,7 +32,6 @@ Used by resmoke.py to run only feature flag tests. import os import sys -from typing import List import typer import yaml @@ -44,7 +43,7 @@ from buildscripts.idl import lib from buildscripts.idl.idl import binder -def get_all_feature_flags_turned_on_by_default(idl_dirs: List[str] = None): +def get_all_feature_flags_turned_on_by_default(idl_dirs: list[str] = None): """Generate a list of all feature flags that default to true.""" all_flags = lib.get_all_feature_flags(idl_dirs) @@ -53,7 +52,7 @@ def get_all_feature_flags_turned_on_by_default(idl_dirs: List[str] = None): ] -def get_all_feature_flags_turned_off_by_default(idl_dirs: List[str] = None): +def get_all_feature_flags_turned_off_by_default(idl_dirs: list[str] = None): """Generate a list of all feature flags that default to false.""" all_flags = lib.get_all_feature_flags(idl_dirs) all_default_false_flags = [ @@ -70,7 +69,7 @@ def get_all_feature_flags_turned_off_by_default(idl_dirs: List[str] = None): return list(set(all_default_false_flags) - set(force_disabled_flags)) -def get_all_unreleased_ifr_feature_flags(idl_dirs: List[str] = None): +def get_all_unreleased_ifr_feature_flags(idl_dirs: list[str] = None): """Generate a list of all features flags in the 'in_development' incremental rollout phase.""" all_flags = lib.get_all_feature_flags(idl_dirs) @@ -81,7 +80,7 @@ def get_all_unreleased_ifr_feature_flags(idl_dirs: List[str] = None): ] -def write_feature_flags_to_file(flags: List[str], filename: str): +def write_feature_flags_to_file(flags: list[str], filename: str): """Helper function to write feature flags to a file.""" with open(filename, "w") as output_file: output_file.write("\n".join(flags)) diff --git a/buildscripts/idl/gen_all_server_params_list.py b/buildscripts/idl/gen_all_server_params_list.py index 44789581a29..a048ed2b176 100644 --- a/buildscripts/idl/gen_all_server_params_list.py +++ b/buildscripts/idl/gen_all_server_params_list.py @@ -32,7 +32,6 @@ Used by DSI to conditionally allow configuration of internalQueryStatsRateLimit import os import sys -from typing import List # Permit imports from "buildscripts". sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../.."))) @@ -41,7 +40,7 @@ from buildscripts.idl import lib from buildscripts.idl.idl import parser -def gen_all_server_params(idl_dirs: List[str] = None): +def gen_all_server_params(idl_dirs: list[str] = None): """Generate a list of all server parameters.""" default_idl_dirs = ["src"] diff --git a/buildscripts/idl/gen_dts.py b/buildscripts/idl/gen_dts.py index 7ddd8b504e4..f42c95d2090 100644 --- a/buildscripts/idl/gen_dts.py +++ b/buildscripts/idl/gen_dts.py @@ -90,7 +90,7 @@ import shutil import sys from pathlib import Path from types import FunctionType -from typing import Dict, Optional, Set, TypedDict +from typing import Optional, TypedDict # Permit imports from "buildscripts". sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../.."))) @@ -100,9 +100,9 @@ from buildscripts.idl.idl.compiler import CompilerImportResolver class Declarations(TypedDict): - structs: Set[str] - commands: Set[str] - enums: Set[str] + structs: set[str] + commands: set[str] + enums: set[str] def object_to_dts(obj: common.SourceLocation): @@ -187,7 +187,7 @@ def object_to_dts(obj: common.SourceLocation): def gen_dts_from_file( input_file: str, existing_decls: Optional[Declarations] = None -) -> Dict[str, int]: +) -> dict[str, int]: if not os.path.exists(input_file): logging.error('File "%s" not found', args.input_file) return {} @@ -204,7 +204,7 @@ def gen_dts( exists_func: FunctionType = os.path.exists, ignore_imported=False, existing_decls: Optional[Declarations] = None, -) -> Dict[str, int]: +) -> dict[str, int]: if parsed_doc.errors: parsed_doc.errors.dump_errors() @@ -245,7 +245,7 @@ def gen_dts_files(): COLS = shutil.get_terminal_size().columns MAX_FILENAME_LEN = COLS // 3 - def print_progress(file, additions: Dict[str, int]) -> None: + def print_progress(file, additions: dict[str, int]) -> None: file = str(file) file = file if len(file) < MAX_FILENAME_LEN else f"{file[:MAX_FILENAME_LEN - 3]}..." info = f"Parsed {file}: ".ljust(MAX_FILENAME_LEN + len("Parsed : ") + 1) diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py index 0e78744f481..36090a5c3b7 100644 --- a/buildscripts/idl/idl/generator.py +++ b/buildscripts/idl/idl/generator.py @@ -36,7 +36,7 @@ import sys import textwrap from abc import ABCMeta, abstractmethod from enum import Enum -from typing import List, cast +from typing import cast from . import ( ast, @@ -552,7 +552,7 @@ class _CppFileWriterBase(object): def get_all_structs(spec: ast.IDLBoundSpec): - return spec.structs + cast(List[ast.Struct], spec.commands) + return spec.structs + cast(list[ast.Struct], spec.commands) def make_mod_tag(vis: str): diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py index c779e961987..d42be7c219a 100644 --- a/buildscripts/idl/idl_check_compatibility.py +++ b/buildscripts/idl/idl_check_compatibility.py @@ -42,7 +42,7 @@ import os import sys from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Optional, Union import yaml from idl import common, errors, parser, syntax @@ -71,17 +71,17 @@ rules = load_rules_file() # Load the subsections from the global "rules.yml" file into separate global variables. # Any of the following assignments will fail if no rules exist for the provided key. -ALLOW_ANY_TYPE_LIST: List[str] = rules["ALLOW_ANY_TYPE_LIST"] -IGNORE_ANY_TO_NON_ANY_LIST: List[str] = rules["IGNORE_ANY_TO_NON_ANY_LIST"] -IGNORE_NON_ANY_TO_ANY_LIST: List[str] = rules["IGNORE_NON_ANY_TO_ANY_LIST"] -ALLOW_CPP_TYPE_CHANGE_LIST: List[str] = rules["ALLOW_CPP_TYPE_CHANGE_LIST"] -IGNORE_STABLE_TO_UNSTABLE_LIST: List[str] = rules["IGNORE_STABLE_TO_UNSTABLE_LIST"] -ALLOWED_STABLE_FIELDS_LIST: List[str] = rules["ALLOWED_STABLE_FIELDS_LIST"] -IGNORE_COMMANDS_LIST: List[str] = rules["IGNORE_COMMANDS_LIST"] +ALLOW_ANY_TYPE_LIST: list[str] = rules["ALLOW_ANY_TYPE_LIST"] +IGNORE_ANY_TO_NON_ANY_LIST: list[str] = rules["IGNORE_ANY_TO_NON_ANY_LIST"] +IGNORE_NON_ANY_TO_ANY_LIST: list[str] = rules["IGNORE_NON_ANY_TO_ANY_LIST"] +ALLOW_CPP_TYPE_CHANGE_LIST: list[str] = rules["ALLOW_CPP_TYPE_CHANGE_LIST"] +IGNORE_STABLE_TO_UNSTABLE_LIST: list[str] = rules["IGNORE_STABLE_TO_UNSTABLE_LIST"] +ALLOWED_STABLE_FIELDS_LIST: list[str] = rules["ALLOWED_STABLE_FIELDS_LIST"] +IGNORE_COMMANDS_LIST: list[str] = rules["IGNORE_COMMANDS_LIST"] RENAMED_COMPLEX_ACCESS_CHECKS: dict[str, str] = rules["RENAMED_COMPLEX_ACCESS_CHECKS"] -ALLOWED_NEW_COMPLEX_ACCESS_CHECKS: dict[str, List[str]] = rules["ALLOWED_NEW_COMPLEX_ACCESS_CHECKS"] -CHANGED_ACCESS_CHECKS_TYPE: dict[str, List[str]] = rules["CHANGED_ACCESS_CHECKS_TYPE"] -ALLOW_FIELD_VALUE_REMOVAL_LIST: dict[str, List[str]] = rules["ALLOW_FIELD_VALUE_REMOVAL_LIST"] +ALLOWED_NEW_COMPLEX_ACCESS_CHECKS: dict[str, list[str]] = rules["ALLOWED_NEW_COMPLEX_ACCESS_CHECKS"] +CHANGED_ACCESS_CHECKS_TYPE: dict[str, list[str]] = rules["CHANGED_ACCESS_CHECKS_TYPE"] +ALLOW_FIELD_VALUE_REMOVAL_LIST: dict[str, list[str]] = rules["ALLOW_FIELD_VALUE_REMOVAL_LIST"] SKIPPED_FILES = [ "unittest.idl", @@ -100,7 +100,7 @@ class AllowedNewPrivilege: """Represents a privilege check that should be ignored by the API compatibility checker.""" resource_pattern: str - action_type: List[str] + action_type: list[str] agg_stage: Optional[str] = None @classmethod @@ -108,7 +108,7 @@ class AllowedNewPrivilege: return cls(privilege.resource_pattern, privilege.action_type, privilege.agg_stage) -ALLOWED_NEW_ACCESS_CHECK_PRIVILEGES: Dict[str, List[AllowedNewPrivilege]] = dict( +ALLOWED_NEW_ACCESS_CHECK_PRIVILEGES: dict[str, list[AllowedNewPrivilege]] = dict( # Do not add any command other than the aggregate command or any privilege that is not required # only by an aggregation stage not present in previously released versions. aggregate=[], @@ -180,12 +180,12 @@ def is_stable(stability: Optional[str]) -> bool: def get_new_commands( - ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: List[str] -) -> Tuple[Dict[str, syntax.Command], Dict[str, syntax.IDLParsedSpec], Dict[str, str]]: + ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: list[str] +) -> tuple[dict[str, syntax.Command], dict[str, syntax.IDLParsedSpec], dict[str, str]]: """Get new IDL commands and check validity.""" - new_commands: Dict[str, syntax.Command] = dict() - new_command_file: Dict[str, syntax.IDLParsedSpec] = dict() - new_command_file_path: Dict[str, str] = dict() + new_commands: dict[str, syntax.Command] = dict() + new_command_file: dict[str, syntax.IDLParsedSpec] = dict() + new_command_file_path: dict[str, str] = dict() for dirpath, _, filenames in os.walk(new_idl_dir): for new_filename in filenames: @@ -265,8 +265,8 @@ def check_subset( cmd_name: str, field_name: str, type_name: str, - sub_list: List[Union[str, syntax.EnumValue]], - super_list: List[Union[str, syntax.EnumValue]], + sub_list: list[Union[str, syntax.EnumValue]], + super_list: list[Union[str, syntax.EnumValue]], file_path: str, ): """Check if sub_list is a subset of the super_list and log an error if not.""" @@ -283,8 +283,8 @@ def check_superset( ctxt: IDLCompatibilityContext, cmd_name: str, type_name: str, - super_list: List[Union[str, syntax.EnumValue]], - sub_list: List[Union[str, syntax.EnumValue]], + super_list: list[Union[str, syntax.EnumValue]], + sub_list: list[Union[str, syntax.EnumValue]], file_path: str, param_name: Optional[str], is_command_parameter: bool, @@ -1165,7 +1165,7 @@ def check_param_or_type_validator( # SERVER-71601. # # Do not add additional parameters to this list. - ignore_validator_check_list: List[str] = [] + ignore_validator_check_list: list[str] = [] if new_field.validator: if old_field.validator: @@ -1493,8 +1493,8 @@ def check_namespace( def check_error_reply( old_basic_types_path: str, new_basic_types_path: str, - old_import_directories: List[str], - new_import_directories: List[str], + old_import_directories: list[str], + new_import_directories: list[str], ) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new ErrorReply.""" old_idl_dir = os.path.dirname(old_basic_types_path) @@ -1546,8 +1546,8 @@ def check_error_reply( def split_complex_checks( - complex_checks: List[syntax.AccessCheck], -) -> Tuple[List[str], List[syntax.Privilege]]: + complex_checks: list[syntax.AccessCheck], +) -> tuple[list[str], list[syntax.Privilege]]: """Split a list of AccessCheck into checks and privileges.""" checks = [x.check for x in complex_checks if x.check is not None] privileges = [x.privilege for x in complex_checks if x.privilege is not None] @@ -1566,8 +1566,8 @@ def map_complex_access_check_name(name: str) -> str: def check_complex_checks( ctxt: IDLCompatibilityContext, - old_complex_checks: List[syntax.AccessCheck], - new_complex_checks: List[syntax.AccessCheck], + old_complex_checks: list[syntax.AccessCheck], + new_complex_checks: list[syntax.AccessCheck], cmd: syntax.Command, new_idl_file_path: str, ) -> None: @@ -1615,10 +1615,10 @@ def check_complex_checks( def split_complex_checks_agg_stages( - complex_checks: List[syntax.AccessCheck], -) -> Dict[str, List[syntax.AccessCheck]]: + complex_checks: list[syntax.AccessCheck], +) -> dict[str, list[syntax.AccessCheck]]: """Split a list of AccessChecks into a map keyed by aggregation stage (defaults to None).""" - complex_checks_agg_stages: Dict[str, List[syntax.AccessCheck]] = dict() + complex_checks_agg_stages: dict[str, list[syntax.AccessCheck]] = dict() for access_check in complex_checks: agg_stage = None if access_check.privilege is not None: @@ -1632,8 +1632,8 @@ def split_complex_checks_agg_stages( def check_complex_checks_agg_stages( ctxt: IDLCompatibilityContext, - old_complex_checks: List[syntax.AccessCheck], - new_complex_checks: List[syntax.AccessCheck], + old_complex_checks: list[syntax.AccessCheck], + new_complex_checks: list[syntax.AccessCheck], cmd: syntax.Command, new_idl_file_path: str, ) -> None: @@ -1711,8 +1711,8 @@ def check_security_access_checks( def check_compatibility( old_idl_dir: str, new_idl_dir: str, - old_import_directories: List[str], - new_import_directories: List[str], + old_import_directories: list[str], + new_import_directories: list[str], ) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new IDL commands.""" ctxt = IDLCompatibilityContext(old_idl_dir, new_idl_dir, IDLCompatibilityErrorCollection()) @@ -1724,7 +1724,7 @@ def check_compatibility( # Check new commands' compatibility with old ones. # Note, a command can be added to V1 at any time, it's ok if a # new command has no corresponding old command. - old_commands: Dict[str, syntax.Command] = dict() + old_commands: dict[str, syntax.Command] = dict() for dirpath, _, filenames in os.walk(old_idl_dir): for old_filename in filenames: if not old_filename.endswith(".idl") or old_filename in SKIPPED_FILES: @@ -1827,11 +1827,11 @@ def check_compatibility( def get_generic_arguments( - gen_args_file_path: str, includes: List[str] -) -> Tuple[Set[str], Set[str]]: + gen_args_file_path: str, includes: list[str] +) -> tuple[set[str], set[str]]: """Get arguments and reply fields from generic_argument.idl and check validity.""" - arguments: Set[str] = set() - reply_fields: Set[str] = set() + arguments: set[str] = set() + reply_fields: set[str] = set() with open(gen_args_file_path) as gen_args_file: parsed_idl_file = parser.parse( @@ -1875,8 +1875,8 @@ def get_generic_arguments( def check_generic_arguments_compatibility( old_gen_args_file_path: str, new_gen_args_file_path: str, - old_includes: List[str], - new_includes: List[str], + old_includes: list[str], + new_includes: list[str], ) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new generic_argument.idl files.""" # IDLCompatibilityContext takes in both 'old_idl_dir' and 'new_idl_dir', diff --git a/buildscripts/idl/idl_compatibility_errors.py b/buildscripts/idl/idl_compatibility_errors.py index bb7f5628dfa..8e8c6df8a1a 100644 --- a/buildscripts/idl/idl_compatibility_errors.py +++ b/buildscripts/idl/idl_compatibility_errors.py @@ -34,7 +34,7 @@ Common error handling code for IDL compatibility checker. import inspect import sys -from typing import List, Optional +from typing import Optional # Public error codes used by IDL compatibility checker. # Used by tests cases to validate expected errors are thrown in negative tests. @@ -188,7 +188,7 @@ class IDLCompatibilityErrorCollection(object): def __init__(self) -> None: """Initialize IDLCompatibilityErrorCollection.""" - self._errors: List[IDLCompatibilityError] = [] + self._errors: list[IDLCompatibilityError] = [] def add( self, @@ -236,11 +236,11 @@ class IDLCompatibilityErrorCollection(object): assert error is not None return error - def get_all_errors_by_command_name(self, command_name: str) -> List[IDLCompatibilityError]: + def get_all_errors_by_command_name(self, command_name: str) -> list[IDLCompatibilityError]: """Get all the errors in the error collection with the command command_name.""" return [a for a in self._errors if a.command_name == command_name] - def to_list(self) -> List[str]: + def to_list(self) -> list[str]: """Return a list of formatted error messages.""" return [str(error) for error in self._errors] diff --git a/buildscripts/idl/lib.py b/buildscripts/idl/lib.py index f8087217c40..93115bc48f2 100644 --- a/buildscripts/idl/lib.py +++ b/buildscripts/idl/lib.py @@ -27,13 +27,12 @@ """Library functions and utility methods used across user-facing IDL scripts.""" import os -from typing import List, Set from buildscripts.idl.idl import parser, syntax from buildscripts.idl.idl.compiler import CompilerImportResolver -def list_idls(directory: str) -> Set[str]: +def list_idls(directory: str) -> set[str]: """Find all IDL files in the current directory.""" return { os.path.join(dirpath, filename) @@ -43,7 +42,7 @@ def list_idls(directory: str) -> Set[str]: } -def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedSpec: +def parse_idl(idl_path: str, import_directories: list[str]) -> syntax.IDLParsedSpec: """Parse an IDL file or throw an error.""" parsed_doc = parser.parse(open(idl_path), idl_path, CompilerImportResolver(import_directories)) @@ -65,7 +64,7 @@ def is_third_party_idl(idl_path: str) -> bool: return False -def get_all_feature_flags(idl_dirs: List[str] = None): +def get_all_feature_flags(idl_dirs: list[str] = None): """Generate a dict of all feature flags with their default value.""" default_idl_dirs = ["src", "buildscripts"] diff --git a/buildscripts/idl/tests/test_gendts.py b/buildscripts/idl/tests/test_gendts.py index 444f3a86837..dd8e8f21fc9 100755 --- a/buildscripts/idl/tests/test_gendts.py +++ b/buildscripts/idl/tests/test_gendts.py @@ -37,7 +37,7 @@ import unittest from contextlib import contextmanager from io import StringIO from types import SimpleNamespace -from typing import DefaultDict, Iterable, Tuple +from typing import Iterable # import package so that it works regardless of whether we run as a module or file if __package__ is None: @@ -48,6 +48,8 @@ else: # Permit imports from "buildscripts". sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../../../"))) +from collections import defaultdict + from buildscripts.idl.gen_dts import gen_dts, object_to_dts, parser, syntax from buildscripts.idl.idl.compiler import CompilerImportResolver @@ -78,7 +80,7 @@ class TestGenDTS(testcase.IDLTestcase): class MockFilesystem: def __init__(self) -> None: - self.files = DefaultDict(lambda: "") + self.files = defaultdict(lambda: "") @contextmanager def open(self, path: str, mode: str): @@ -120,13 +122,13 @@ class TestGenDTS(testcase.IDLTestcase): self.assertEqual(mock_fs.files["src/mongo/shell/enums_gen.d.ts"], "\n".join(enums)) self.assertEqual(mock_fs.files["src/mongo/shell/commands_gen.d.ts"], "\n".join(commands)) - def _get_generic_idl_with_header(self) -> Tuple[str]: + def _get_generic_idl_with_header(self) -> tuple[str]: """Returns a header necessary to use commands in an IDL.""" return ("global:", " cpp_namespace: 'mongo'", "", "imports: []", "") def _get_command_idl_text( self, name: str, description: str, namespace: str = "ignored" - ) -> Tuple[str]: + ) -> tuple[str]: """Returns a command IDL declaration with a given name, description, and namespace.""" return ( "commands:", @@ -140,7 +142,7 @@ class TestGenDTS(testcase.IDLTestcase): def _get_command_idl_with_header( self, name: str, description: str, namespace: str = "ignored" - ) -> Tuple[str]: + ) -> tuple[str]: """Returns a command IDL declaration with a given name, description, and namespace, with the necessary headers.""" return ( "global:", diff --git a/buildscripts/jepsen_report.py b/buildscripts/jepsen_report.py index 09df6265b23..e3190afc017 100644 --- a/buildscripts/jepsen_report.py +++ b/buildscripts/jepsen_report.py @@ -5,7 +5,7 @@ import os import re import sys from datetime import datetime, timezone -from typing import List, Optional, Tuple +from typing import Optional import click from typing_extensions import TypedDict @@ -16,10 +16,10 @@ from buildscripts.simple_report import Report, Result class ParserOutput(TypedDict): """Result of parsing jepsen log file. Each List[str] is a list of test names.""" - success: List[str] - unknown: List[str] - crashed: List[str] - failed: List[str] + success: list[str] + unknown: list[str] + crashed: list[str] + failed: list[str] start: int end: int elapsed: int @@ -41,7 +41,7 @@ def _time_parse(time: str): return date.replace(microsecond=microseconds, tzinfo=timezone.utc) -def _calc_time_from_log(log: str) -> Tuple[int, int, int]: +def _calc_time_from_log(log: str) -> tuple[int, int, int]: if not log: return (0, 0, 0) start_time = None @@ -67,7 +67,7 @@ UNKNOWN_RE = re.compile("([0-9]+) unknown") FAIL_RE = re.compile("([0-9]+) failures") -def parse(text: List[str]) -> ParserOutput: # noqa: D406 +def parse(text: list[str]) -> ParserOutput: # noqa: D406 """Given a List of strings representing jepsen log file split by newlines, return the ParserOutput struct. Args: @@ -80,10 +80,10 @@ def parse(text: List[str]) -> ParserOutput: # noqa: D406 AssertionError: If there is a mismatch between the count of matches and the corresponding test list. """ - successful_tests: List[str] = [] - indeterminate_tests: List[str] = [] - crashed_tests: List[str] = [] - failed_tests: List[str] = [] + successful_tests: list[str] = [] + indeterminate_tests: list[str] = [] + crashed_tests: list[str] = [] + failed_tests: list[str] = [] target = None success_table_matches = 0 unknown_table_matches = 0 @@ -258,7 +258,7 @@ def report( ) -def _get_log_lines(filename: str) -> List[str]: +def _get_log_lines(filename: str) -> list[str]: with open(filename) as fh: return fh.read().splitlines() diff --git a/buildscripts/large_file_check.py b/buildscripts/large_file_check.py index d22c4b1814e..379b388ed7c 100755 --- a/buildscripts/large_file_check.py +++ b/buildscripts/large_file_check.py @@ -8,7 +8,7 @@ import os import pathlib import sys import textwrap -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional import structlog from git import Repo @@ -27,7 +27,7 @@ from buildscripts.patch_builds.change_data import ( # Console renderer for structured logging -def renderer(_logger: logging.Logger, _name: str, eventdict: Dict[Any, Any]) -> str: +def renderer(_logger: logging.Logger, _name: str, eventdict: dict[Any, Any]) -> str: if "files" in eventdict: return "{event}: {files}".format(**eventdict) if "repo" in eventdict: @@ -54,7 +54,7 @@ LOGGER = structlog.get_logger(__name__) MONGO_REVISION_ENV_VAR = "REVISION" -def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]: +def _get_repos_and_revisions() -> tuple[list[Repo], RevisionMap]: """Get the repo object and a map of revisions to compare against.""" repo_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", None) @@ -67,7 +67,7 @@ def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]: return repos, revision_map -def git_changed_files(excludes: List[pathlib.Path]) -> List[pathlib.Path]: +def git_changed_files(excludes: list[pathlib.Path]) -> list[pathlib.Path]: """ Get the files that have changes since the last git commit. @@ -95,7 +95,7 @@ def git_changed_files(excludes: List[pathlib.Path]) -> List[pathlib.Path]: return files -def diff_file_sizes(size_limit: int, excludes: Optional[List[str]] = None) -> List[pathlib.Path]: +def diff_file_sizes(size_limit: int, excludes: Optional[list[str]] = None) -> list[pathlib.Path]: if excludes is None: excludes = [] diff --git a/buildscripts/legacy_commands_check.py b/buildscripts/legacy_commands_check.py index b4be2adf603..4b0444c084b 100755 --- a/buildscripts/legacy_commands_check.py +++ b/buildscripts/legacy_commands_check.py @@ -4,7 +4,6 @@ import os import re import sys -from typing import List, Tuple # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -21,7 +20,7 @@ LEGACY_TYPES = [ ] -def check_file_for_legacy_type(modified_lines: List[Tuple[int, str]]) -> bool: +def check_file_for_legacy_type(modified_lines: list[tuple[int, str]]) -> bool: """Return false if a file defines a legacy command.""" file_has_legacy_type = False diff --git a/buildscripts/lint_markdown_links.py b/buildscripts/lint_markdown_links.py index 38d6b20b17e..b3eb39b6ced 100755 --- a/buildscripts/lint_markdown_links.py +++ b/buildscripts/lint_markdown_links.py @@ -120,7 +120,7 @@ import re import sys import urllib.parse from dataclasses import dataclass -from typing import Iterable, List, Optional, Tuple +from typing import Iterable, Optional HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$") HTML_ANCHOR_RE = re.compile(r'\s*?', re.IGNORECASE) @@ -273,8 +273,8 @@ def is_http_url(url: str) -> bool: return url.startswith("http://") or url.startswith("https://") -def find_markdown_files(root: str) -> List[str]: - files: List[str] = [] +def find_markdown_files(root: str) -> list[str]: + files: list[str] = [] for dirpath, _, filenames in os.walk(root): for fn in filenames: if fn.lower().endswith(".md"): @@ -282,8 +282,8 @@ def find_markdown_files(root: str) -> List[str]: return files -def parse_links(file_path: str) -> List[Tuple[int, str, str]]: - links: List[Tuple[int, str, str]] = [] +def parse_links(file_path: str) -> list[tuple[int, str, str]]: + links: list[tuple[int, str, str]] = [] try: with open(file_path, "r", encoding="utf-8") as f: in_fence = False @@ -546,11 +546,11 @@ def validate_link(current_file: str, line: int, text: str, target: str) -> Optio return None -def lint_files(files: Iterable[str], workers: int) -> List[LinkIssue]: - issues: List[LinkIssue] = [] +def lint_files(files: Iterable[str], workers: int) -> list[LinkIssue]: + issues: list[LinkIssue] = [] - def process(file_path: str) -> List[LinkIssue]: - file_issues: List[LinkIssue] = [] + def process(file_path: str) -> list[LinkIssue]: + file_issues: list[LinkIssue] = [] links = parse_links(file_path) for line, text, target in links: issue = validate_link(file_path, line, text, target) @@ -566,7 +566,7 @@ def lint_files(files: Iterable[str], workers: int) -> List[LinkIssue]: return issues -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: ap = argparse.ArgumentParser(description="Markdown link linter for src/mongo markdown files.") ap.add_argument("--root", default="src/mongo", help="Root directory to scan") ap.add_argument( @@ -646,7 +646,7 @@ def main(argv: List[str]) -> int: fix_count = 0 # Group issues by file for editing - issues_by_file: dict[str, List[LinkIssue]] = {} + issues_by_file: dict[str, list[LinkIssue]] = {} for iss in issues: issues_by_file.setdefault(iss.file, []).append(iss) @@ -677,7 +677,7 @@ def main(argv: List[str]) -> int: # Deduplicate identical (message, target) to avoid repeated work (retain first occurrence) seen_sig = set() - deduped: List[LinkIssue] = [] + deduped: list[LinkIssue] = [] for iss in file_issues: sig = (iss.message, iss.target) if sig in seen_sig: diff --git a/buildscripts/linter/filediff.py b/buildscripts/linter/filediff.py index 1fc2d00cc7a..e27b6e6773e 100644 --- a/buildscripts/linter/filediff.py +++ b/buildscripts/linter/filediff.py @@ -2,7 +2,7 @@ import os import sys -from typing import Callable, Dict, List, Tuple +from typing import Callable import structlog from git import Repo @@ -23,7 +23,7 @@ LOGGER = structlog.get_logger(__name__) MONGO_REVISION_ENV_VAR = "REVISION" -def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]: +def _get_repos_and_revisions() -> tuple[list[Repo], RevisionMap]: """Get the repo object and a map of revisions to compare against.""" repos = [Repo(git.get_base_dir())] @@ -42,7 +42,7 @@ def _filter_file(filename: str, is_interesting_file: Callable[[str], bool]) -> b return os.path.exists(filename) and is_interesting_file(filename) -def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> List[str]: +def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> list[str]: """ Get the files that have changes since the last git commit. @@ -63,7 +63,7 @@ def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> def gather_changed_files_with_lines( is_interesting_file: Callable[[str], bool], -) -> Dict[str, List[Tuple[int, str]]]: +) -> dict[str, list[tuple[int, str]]]: """ Get the files that have changes since the last git commit, along with details of the specific lines that have changed. diff --git a/buildscripts/linter/git.py b/buildscripts/linter/git.py index 4abc223b6ea..989af328391 100644 --- a/buildscripts/linter/git.py +++ b/buildscripts/linter/git.py @@ -6,7 +6,7 @@ import itertools import os import re from pathlib import Path -from typing import Callable, List +from typing import Callable from buildscripts.linter import git_base as _git @@ -29,7 +29,7 @@ def get_base_dir(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -def get_repos() -> List[Repo]: +def get_repos() -> list[Repo]: """Get a list of Repos to check linters for.""" return [Repo(get_base_dir())] diff --git a/buildscripts/linter/pyrightlinter.py b/buildscripts/linter/pyrightlinter.py index 6761d29bd89..0b12c30118b 100644 --- a/buildscripts/linter/pyrightlinter.py +++ b/buildscripts/linter/pyrightlinter.py @@ -1,7 +1,5 @@ """Pyright linter support module.""" -from typing import List - from . import base @@ -12,10 +10,10 @@ class PyrightLinter(base.LinterBase): """Create a Pyright linter.""" super(PyrightLinter, self).__init__("pyright", "1.1.393") - def get_lint_version_cmd_args(self) -> List[str]: + def get_lint_version_cmd_args(self) -> list[str]: """Get the command to run a version check.""" return ["--version"] - def get_lint_cmd_args(self, files: List[str]) -> List[str]: + def get_lint_cmd_args(self, files: list[str]) -> list[str]: """Get the command to run a check.""" return files if files else ["."] diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 933fdc43208..2f581454fef 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -29,7 +29,7 @@ from abc import ABC, abstractmethod from collections import OrderedDict from datetime import timedelta from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union import requests from tenacity import Retrying, retry_if_result, stop_after_delay, wait_fixed @@ -52,7 +52,7 @@ class DbgFileResolver(ABC): """Base gdb path resolver class.""" @abstractmethod - def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]: + def get_dbg_file(self, soinfo: dict[str, Any]) -> Union[str, None]: """ To get path for given build info. @@ -124,7 +124,7 @@ class CachedResults(object): Idea is to allow only N items to be present in cache at a time and eliminate extra items on the go. """ - def __init__(self, max_cache_size: int, initial_cache: Dict[str, str] = None): + def __init__(self, max_cache_size: int, initial_cache: dict[str, str] = None): """ Initialize instance. @@ -134,7 +134,7 @@ class CachedResults(object): self._max_cache_size = max_cache_size self._cached_results = OrderedDict(initial_cache or {}) - def insert(self, key: str, value: str) -> Dict[str, str] or None: + def insert(self, key: str, value: str) -> dict[str, str] or None: """ Insert new data into cache. @@ -302,7 +302,7 @@ class PathResolver(DbgFileResolver): """ return self._cached_results.get(key) - def add_to_cache(self, key: str, value: str) -> Dict[str, str]: + def add_to_cache(self, key: str, value: str) -> dict[str, str]: """ Add new value to cache. @@ -484,7 +484,7 @@ def parse_input(trace_doc, dbg_path_resolver): return frames -def get_version(trace_doc: Dict[str, Any]) -> Optional[str]: +def get_version(trace_doc: dict[str, Any]) -> Optional[str]: """ Get version from trace doc. @@ -569,8 +569,8 @@ def symbolize_frames( def preprocess_frames( - dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], input_format: str -) -> List[Dict[str, Any]]: + dbg_path_resolver: DbgFileResolver, trace_doc: dict[str, Any], input_format: str +) -> list[dict[str, Any]]: """ Process the paths in frame objects. @@ -592,7 +592,7 @@ def preprocess_frames( return frames -def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool: +def has_high_not_found_paths_ratio(frames: list[dict[str, Any]]) -> bool: """ Check whether not found paths in frames ratio is higher than 0.5. @@ -606,10 +606,10 @@ def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool: def preprocess_frames_with_retries( dbg_path_resolver: DbgFileResolver, - trace_doc: Dict[str, Any], + trace_doc: dict[str, Any], input_format: str, total_seconds_for_retries: int = 0, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ Process the paths in frame objects. diff --git a/buildscripts/monitor_build_status/cli.py b/buildscripts/monitor_build_status/cli.py index f3f4c732992..28e2942d5cb 100644 --- a/buildscripts/monitor_build_status/cli.py +++ b/buildscripts/monitor_build_status/cli.py @@ -4,7 +4,6 @@ import os import sys from datetime import datetime, timedelta, timezone from enum import Enum -from typing import Dict, List, Tuple import structlog import typer @@ -44,7 +43,7 @@ class CodeMergeStatus(Enum): GREEN = "GREEN" @classmethod - def from_threshold_percentages(cls, threshold_percentages: List[float]) -> CodeMergeStatus: + def from_threshold_percentages(cls, threshold_percentages: list[float]) -> CodeMergeStatus: if any(percentage > 100 for percentage in threshold_percentages): return cls.RED return cls.GREEN @@ -75,7 +74,7 @@ class MonitorBuildStatusOrchestrator: summaries = "" for scopes_config in notification_config.scopes: - scope_percentages: Dict[str, List[float]] = {} + scope_percentages: dict[str, list[float]] = {} issue_report = self._make_report(scopes_config) issue_count_status_msg, issue_count_percentages = self._get_issue_counts_status( @@ -117,9 +116,9 @@ class MonitorBuildStatusOrchestrator: def _get_issue_counts_status( self, scope_name: str, issue_report: IssueReport, notification_config: NotificationsConfig - ) -> Tuple[str, Dict[str, List[float]]]: + ) -> tuple[str, dict[str, list[float]]]: now = datetime.now(timezone.utc) - percentages: Dict[str, List[float]] = {} + percentages: dict[str, list[float]] = {} headers = [scope_name, "Hot Issues", "Cold Issues"] table_data = [] @@ -129,7 +128,7 @@ class MonitorBuildStatusOrchestrator: hot_issue_count: int, cold_issue_count: int, thresholds: IssueThresholds, - slack_tags: List[str], + slack_tags: list[str], ) -> None: if all(count == 0 for count in [hot_issue_count, cold_issue_count]): return @@ -242,7 +241,7 @@ class MonitorBuildStatusOrchestrator: return message, percentages @staticmethod - def _summarize(scope_name: str, scope_percentages: Dict[str, List[float]]) -> str: + def _summarize(scope_name: str, scope_percentages: dict[str, list[float]]) -> str: summary = f"`SUMMARY [{scope_name}]`" red_sub_scopes = [] diff --git a/buildscripts/monitor_build_status/code_lockdown_config.py b/buildscripts/monitor_build_status/code_lockdown_config.py index 368907e3dc7..98ae4aa5556 100644 --- a/buildscripts/monitor_build_status/code_lockdown_config.py +++ b/buildscripts/monitor_build_status/code_lockdown_config.py @@ -1,7 +1,7 @@ from __future__ import annotations from copy import deepcopy -from typing import List, Optional +from typing import Optional import yaml from pydantic import BaseModel @@ -40,33 +40,33 @@ class ScopesConfig(BaseModel): class SlackConfig(BaseModel): channel: str - overall_scope_tags: List[str] + overall_scope_tags: list[str] message_footer: str short_issue_data_table: bool = False class NotificationsConfig(BaseModel): - scopes: List[ScopesConfig] + scopes: list[ScopesConfig] thresholds: ThresholdsConfig slack: SlackConfig class TeamConfig(BaseModel): name: str - slack_tags: Optional[List[str]] + slack_tags: Optional[list[str]] thresholds: Optional[ThresholdOverride] class GroupConfig(BaseModel): name: str - teams: List[str] - slack_tags: Optional[List[str]] + teams: list[str] + slack_tags: Optional[list[str]] class CodeLockdownConfig(BaseModel): - notifications: List[NotificationsConfig] - teams: List[TeamConfig] - groups: List[GroupConfig] + notifications: list[NotificationsConfig] + teams: list[TeamConfig] + groups: list[GroupConfig] @classmethod def from_yaml_config(cls, file_path: str) -> CodeLockdownConfig: @@ -79,11 +79,11 @@ class CodeLockdownConfig(BaseModel): with open(file_path, encoding="utf8") as file_handler: return cls(**yaml.safe_load(file_handler)) - def get_all_group_names(self) -> List[str]: + def get_all_group_names(self) -> list[str]: """Get all group names.""" return [group.name for group in self.groups] - def get_group_teams(self, group_name: str) -> List[str]: + def get_group_teams(self, group_name: str) -> list[str]: """ Get group teams. @@ -96,7 +96,7 @@ class CodeLockdownConfig(BaseModel): return [] - def get_group_slack_tags(self, group_name: str) -> List[str]: + def get_group_slack_tags(self, group_name: str) -> list[str]: """ Get group slack tags. @@ -109,7 +109,7 @@ class CodeLockdownConfig(BaseModel): return [] - def get_team_slack_tags(self, team_name: str) -> List[str]: + def get_team_slack_tags(self, team_name: str) -> list[str]: """ Get team slack tags. diff --git a/buildscripts/monitor_build_status/issue_report.py b/buildscripts/monitor_build_status/issue_report.py index 39f7bfb117a..02957c0fe8e 100644 --- a/buildscripts/monitor_build_status/issue_report.py +++ b/buildscripts/monitor_build_status/issue_report.py @@ -2,7 +2,7 @@ from __future__ import annotations from datetime import datetime, timezone from enum import Enum -from typing import Dict, Iterable, List, NamedTuple, Optional, Set +from typing import Iterable, NamedTuple, Optional from buildscripts.monitor_build_status.jira_service import IssueTuple @@ -13,8 +13,8 @@ class IssueCategory(str, Enum): class CategorizedIssues(NamedTuple): - hot: Set[IssueTuple] - cold: Set[IssueTuple] + hot: set[IssueTuple] + cold: set[IssueTuple] @classmethod def empty(cls) -> CategorizedIssues: @@ -49,7 +49,7 @@ class CategorizedIssues(NamedTuple): class IssueReport(NamedTuple): - team_reports: Dict[str, CategorizedIssues] + team_reports: dict[str, CategorizedIssues] @classmethod def empty(cls) -> IssueReport: @@ -76,7 +76,7 @@ class IssueReport(NamedTuple): self, category: IssueCategory, include_items_older_than_time: Optional[datetime] = None, - assigned_teams: Optional[List[str]] = None, + assigned_teams: Optional[list[str]] = None, ) -> int: """ Calculate Issue count for a given criteria. diff --git a/buildscripts/monitor_build_status/jira_service.py b/buildscripts/monitor_build_status/jira_service.py index a4047a4fded..66c591fb50f 100644 --- a/buildscripts/monitor_build_status/jira_service.py +++ b/buildscripts/monitor_build_status/jira_service.py @@ -2,7 +2,7 @@ from __future__ import annotations from datetime import datetime from enum import Enum -from typing import Any, List, NamedTuple, Optional +from typing import Any, NamedTuple, Optional import dateutil.parser from jira import Issue @@ -63,7 +63,7 @@ class JiraService: def __init__(self, jira_client: JiraClient) -> None: self.jira_client = jira_client - def fetch_issues(self, query: str) -> List[IssueTuple]: + def fetch_issues(self, query: str) -> list[IssueTuple]: """ Fetch issues from Jira and transform it into consumable form. diff --git a/buildscripts/monitor_mongo_fork_10gen.py b/buildscripts/monitor_mongo_fork_10gen.py index ddba43e9632..4d3a6b9598e 100644 --- a/buildscripts/monitor_mongo_fork_10gen.py +++ b/buildscripts/monitor_mongo_fork_10gen.py @@ -1,6 +1,6 @@ import argparse import sys -from typing import List, Optional +from typing import Optional from github import Github, GithubException, GithubIntegration from simple_report import make_report, put_report @@ -50,7 +50,7 @@ def get_users_who_forked_mongo_repo(owner: str, repo: str, token: str) -> list[s return [fork.owner.login for fork in repository.get_forks() if not fork.archived] -def are_users_members_of_org(users: List[str], org: str, token: str) -> List[str]: # noqa: D406 +def are_users_members_of_org(users: list[str], org: str, token: str) -> list[str]: # noqa: D406 """ Check if users are members of a particular organization. diff --git a/buildscripts/package_test.py b/buildscripts/package_test.py index 8e5f4024c68..f475beea51d 100644 --- a/buildscripts/package_test.py +++ b/buildscripts/package_test.py @@ -13,7 +13,7 @@ import traceback import uuid from concurrent import futures from pathlib import Path -from typing import Any, Dict, Generator, List, Optional, Set, Tuple +from typing import Any, Generator, Optional from urllib.parse import urlparse import docker @@ -313,10 +313,10 @@ OS_DOCKER_LOOKUP = { } # These versions are marked "current" but in fact are EOL -VERSIONS_TO_SKIP: Set[str] = set( +VERSIONS_TO_SKIP: set[str] = set( ["3.0.15", "3.2.22", "3.4.24", "3.6.23", "4.0.28", "4.2.24", "4.2.25", "4.4.29", "6.3.2"] ) -DISABLED_TESTS: Set[Tuple[str, str]] = set() +DISABLED_TESTS: set[tuple[str, str]] = set() VALID_TAR_DIRECTORY_ARCHITECTURES = [ "linux-aarch64", @@ -340,10 +340,10 @@ class Test: package_manager: str = dataclasses.field(default="", repr=False) update_command: str = dataclasses.field(default="", repr=False) install_command: str = dataclasses.field(default="", repr=False) - base_packages: List[str] = dataclasses.field(default_factory=list) + base_packages: list[str] = dataclasses.field(default_factory=list) python_command: str = dataclasses.field(default="", repr=False) - packages_urls: List[str] = dataclasses.field(default_factory=list) - packages_paths: List[Path] = dataclasses.field(default_factory=list) + packages_urls: list[str] = dataclasses.field(default_factory=list) + packages_paths: list[Path] = dataclasses.field(default_factory=list) attempts: int = dataclasses.field(default=0) def __post_init__(self) -> None: @@ -388,7 +388,7 @@ def get_image(test: Test, client: DockerClient) -> Image: time.sleep(1) -def join_commands(commands: List[str], sep: str = " && ") -> str: +def join_commands(commands: list[str], sep: str = " && ") -> str: return sep.join(commands) @@ -421,7 +421,7 @@ def run_test(test: Test, client: DockerClient) -> Result: test_external_root = Path(__file__).parent.resolve() logging.debug(test_external_root) log_external_path = Path.joinpath(test_external_root, log_name) - commands: List[str] = ["export PYTHONIOENCODING=UTF-8"] + commands: list[str] = ["export PYTHONIOENCODING=UTF-8"] if test.os_name.startswith("rhel"): # RHEL distros need EPEL for Compass dependencies @@ -552,7 +552,7 @@ r = retry_call( mongosh_releases = r.json() -def iterate_over_downloads() -> Generator[Dict[str, Any], None, None]: +def iterate_over_downloads() -> Generator[dict[str, Any], None, None]: # TODO: TOOLS-3204 - we need to sub the arch alias until package # rchitectures are named consistently with the server packages for version in current_releases["versions"]: @@ -615,7 +615,7 @@ def get_mongosh_package(arch_name: str, os_name: str) -> Optional[str]: return None -def get_arch_aliases(arch_name: str) -> List[str]: +def get_arch_aliases(arch_name: str) -> list[str]: if arch_name in ("amd64", "x86_64"): return ["amd64", "x86_64"] if arch_name in ("ppc64le", "ppc64el"): @@ -662,10 +662,10 @@ def validate_no_libdwarf(sources_text, edition, binfile): raise Exception(f"Found LGPL code from libdwarf in {edition} binary {binfile}.") -arches: Set[str] = set() -oses: Set[str] = set() -editions: Set[str] = set() -versions: Set[str] = set() +arches: set[str] = set() +oses: set[str] = set() +editions: set[str] = set() +versions: set[str] = set() for dl in iterate_over_downloads(): editions.add(get_edition_alias(dl["edition"])) @@ -765,8 +765,8 @@ arch: str = args.arch if arch == "auto": arch = platform.machine() -tests: List[Test] = [] -urls: List[str] = [] +tests: list[Test] = [] +urls: list[str] = [] if args.command == "branch": # If evg-build-id is provided, download the packages locally using the Evergreen API @@ -916,7 +916,7 @@ if args.command == "release": continue test_os: str = dl["target"] - urls: List[str] = dl["packages"] + urls: list[str] = dl["packages"] server_version: str = dl["version"] edition: str diff --git a/buildscripts/package_test_internal.py b/buildscripts/package_test_internal.py index fd265192310..4b70bfc28b5 100644 --- a/buildscripts/package_test_internal.py +++ b/buildscripts/package_test_internal.py @@ -16,7 +16,7 @@ import tarfile import time import traceback from logging.handlers import WatchedFileHandler -from typing import Dict, List, Optional, Tuple, Union +from typing import Optional, Union root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -39,7 +39,7 @@ JOURNALCTL_URL = ( DOCKER_SYSTEMCTL_REPO + "/eb2a963a7d8413119b432bcb6151af6076b65f84/files/docker/journalctl3.py" ) -TestArgs = Dict[str, Union[str, int, List[str]]] +TestArgs = dict[str, Union[str, int, list[str]]] def run_and_log(cmd: str, end_on_error: bool = True): @@ -53,7 +53,7 @@ def run_and_log(cmd: str, end_on_error: bool = True): return proc -def download_extract_package(package: str) -> List[str]: +def download_extract_package(package: str) -> list[str]: # Handle local files (file:// protocol) - these are pre-downloaded by package_test.py # when using --evg-build-id for private artifacts if package.startswith("file://"): @@ -82,7 +82,7 @@ def download_extract_package(package: str) -> List[str]: return extracted_paths -def download_extract_all_packages(package_urls: List[str]) -> List[str]: +def download_extract_all_packages(package_urls: list[str]) -> list[str]: all_packages = [] # type: List[str] for package_url in package_urls: if package_url: @@ -91,17 +91,17 @@ def download_extract_all_packages(package_urls: List[str]) -> List[str]: return all_packages -def run_apt_test(packages: List[str]): +def run_apt_test(packages: list[str]): logging.info("Detected apt running test.") run_and_log("DEBIAN_FRONTEND=noninteractive apt-get install -y {}".format(" ".join(packages))) -def run_yum_test(packages: List[str]): +def run_yum_test(packages: list[str]): logging.info("Detected yum running test.") run_and_log("yum install -y {}".format(" ".join(packages))) -def run_zypper_test(packages: List[str]): +def run_zypper_test(packages: list[str]): logging.info("Detected zypper running test.") run_and_log("zypper -n --no-gpg-checks install {}".format(" ".join(packages))) @@ -143,7 +143,7 @@ def run_mongo_query(shell, query, should_fail=False, tries=60, interval=1.0): raise RuntimeError("Query retries exceeded, failing test.") -def parse_os_release(path: str) -> Dict[str, str]: +def parse_os_release(path: str) -> dict[str, str]: result = {} # type: Dict[str, str] with open(path, "r", encoding="utf-8") as os_release: for line in os_release: @@ -156,7 +156,7 @@ def parse_os_release(path: str) -> Dict[str, str]: return result -def get_os_release() -> Tuple[str, int, int]: +def get_os_release() -> tuple[str, int, int]: if os.path.exists("/etc/os-release"): release_info = parse_os_release("/etc/os-release") elif os.path.exists("/usr/lib/os-release"): @@ -177,7 +177,7 @@ def get_os_release() -> Tuple[str, int, int]: return os_name, os_version_major, os_version_minor -def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]: +def parse_ulimits(pid: int) -> dict[str, tuple[int, int, Optional[str]]]: ulimit_line_re = re.compile( r"(?P.*?)\s{2,}(?P\S+)\s+(?P\S+)(?:\s+(?P\S+))?", re.MULTILINE ) @@ -205,7 +205,7 @@ def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]: return result -def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs: +def get_test_args(package_manager: str, package_files: list[str]) -> TestArgs: # Set up data for later tests test_args = {} # type: TestArgs diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py index f715f2026f1..83263f0c762 100644 --- a/buildscripts/patch_builds/change_data.py +++ b/buildscripts/patch_builds/change_data.py @@ -2,14 +2,14 @@ import os from itertools import chain -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple +from typing import Any, Iterable, Optional import structlog from git import DiffIndex, Repo LOGGER = structlog.get_logger(__name__) -RevisionMap = Dict[str, str] +RevisionMap = dict[str, str] def _get_id_from_repo(repo: Repo) -> str: @@ -24,7 +24,7 @@ def _get_id_from_repo(repo: Repo) -> str: return os.path.basename(repo.working_dir) -def generate_revision_map(repos: List[Repo], revisions_data: Dict[str, str]) -> RevisionMap: +def generate_revision_map(repos: list[Repo], revisions_data: dict[str, str]) -> RevisionMap: """ Generate a revision map for the given repositories using the revisions in the given file. @@ -49,7 +49,7 @@ def _paths_for_iter(diff, iter_type): return a_path_changes.union(b_path_changes) -def _modified_files_for_diff(diff: DiffIndex, log: Any) -> Set: +def _modified_files_for_diff(diff: DiffIndex, log: Any) -> set: """ Get the set of files modified in the given git diff. @@ -72,7 +72,7 @@ def _modified_files_for_diff(diff: DiffIndex, log: Any) -> Set: return modified_files.union(added_files).union(renamed_files).union(deleted_files) -def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) -> Set[str]: +def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) -> set[str]: """ Find files that were new or added to the repository between commits. @@ -104,7 +104,7 @@ def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) - def find_changed_files_in_repos( repos: Iterable[Repo], revision_map: Optional[RevisionMap] = None -) -> Set[str]: +) -> set[str]: """ Find the changed files. @@ -118,8 +118,8 @@ def find_changed_files_in_repos( def find_modified_lines_for_files( - repo: Repo, changed_files: List[str], revision_map: Optional[RevisionMap] = None -) -> Dict[str, List[Tuple[int, str]]]: + repo: Repo, changed_files: list[str], revision_map: Optional[RevisionMap] = None +) -> dict[str, list[tuple[int, str]]]: """ For each changed file, determine which lines were modified. @@ -164,8 +164,8 @@ def find_modified_lines_for_files( def find_modified_lines_for_files_in_repos( - repos: Iterable[Repo], changed_files: List[str], revision_map: Optional[RevisionMap] = None -) -> Dict[str, List[Tuple[int, str]]]: + repos: Iterable[Repo], changed_files: list[str], revision_map: Optional[RevisionMap] = None +) -> dict[str, list[tuple[int, str]]]: """ Find the modified lines in files with changes. diff --git a/buildscripts/powercycle_sentinel.py b/buildscripts/powercycle_sentinel.py index f16432019f1..abf4bcaced3 100755 --- a/buildscripts/powercycle_sentinel.py +++ b/buildscripts/powercycle_sentinel.py @@ -9,7 +9,6 @@ import os import sys import time from datetime import datetime, timezone -from typing import List import click import structlog @@ -42,7 +41,7 @@ def get_evergreen_api() -> EvergreenApi: sys.exit(1) -def watch_tasks(task_ids: List[str], evg_api: EvergreenApi, watch_interval_secs: int) -> List[str]: +def watch_tasks(task_ids: list[str], evg_api: EvergreenApi, watch_interval_secs: int) -> list[str]: """Watch tasks if they run longer than exec timeout.""" watch_task_ids = task_ids[:] long_running_task_ids = [] @@ -66,7 +65,7 @@ def watch_tasks(task_ids: List[str], evg_api: EvergreenApi, watch_interval_secs: return long_running_task_ids -def get_links(task_ids: List[str]) -> str: +def get_links(task_ids: list[str]) -> str: """Return evergreen task urls delimited by newline.""" return "\n".join([f"{EVERGREEN_HOST}/task/{task_id}" for task_id in task_ids]) diff --git a/buildscripts/pyrightlint.py b/buildscripts/pyrightlint.py index ea2874874de..1e488120cb5 100755 --- a/buildscripts/pyrightlint.py +++ b/buildscripts/pyrightlint.py @@ -6,7 +6,6 @@ import logging import os import subprocess import sys -from typing import List import structlog @@ -27,7 +26,7 @@ def is_interesting_file(filename: str) -> bool: return filename.endswith(".py") -def lint(paths: List[str]): +def lint(paths: list[str]): """Lint specified paths (files or directories) using Pyright.""" if "BUILD_WORKSPACE_DIRECTORY" in os.environ: subprocess.run( diff --git a/buildscripts/quickmongolint.py b/buildscripts/quickmongolint.py index de3e78509eb..46051219d82 100755 --- a/buildscripts/quickmongolint.py +++ b/buildscripts/quickmongolint.py @@ -6,7 +6,6 @@ import logging import os import re import sys -from typing import List # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -33,7 +32,7 @@ def is_interesting_file(file_name: str) -> bool: ) and FILES_RE.search(file_name) -def _lint_files(file_names: List[str]) -> None: +def _lint_files(file_names: list[str]) -> None: """Lint a list of files with clang-format.""" def run_lint1(param1): @@ -53,7 +52,7 @@ def lint_patch(file_name: str) -> None: _lint_files(file_names) -def lint(file_names: List[str]) -> None: +def lint(file_names: list[str]) -> None: # type: (str, Dict[str, str], List[str]) -> None """Lint files command entry point.""" all_file_names = git.get_files_to_check(file_names, is_interesting_file) @@ -61,14 +60,14 @@ def lint(file_names: List[str]) -> None: _lint_files(all_file_names) -def lint_all(file_names: List[str]) -> None: +def lint_all(file_names: list[str]) -> None: """Lint files command entry point based on working tree.""" all_file_names = git.get_files_to_check_working_tree(is_interesting_file) _lint_files(all_file_names) -def lint_my(origin_branch: List[str]) -> None: +def lint_my(origin_branch: list[str]) -> None: """Lint files command based on local changes.""" files = git.get_my_files_to_check(is_interesting_file, origin_branch) files = [f for f in files if os.path.exists(f)] diff --git a/buildscripts/resmoke_proxy/resmoke_proxy.py b/buildscripts/resmoke_proxy/resmoke_proxy.py index 07044eefdbd..a65e1b0f926 100644 --- a/buildscripts/resmoke_proxy/resmoke_proxy.py +++ b/buildscripts/resmoke_proxy/resmoke_proxy.py @@ -1,6 +1,6 @@ """A service to proxy requests to resmoke.""" -from typing import Any, Dict, List +from typing import Any import inject @@ -17,7 +17,7 @@ class ResmokeProxyService: _parser.set_run_options(run_options) self._suite_config = _suiteconfig - def list_tests(self, suite_name: str) -> List[str]: + def list_tests(self, suite_name: str) -> list[str]: """ List the test files that are part of the suite. @@ -35,7 +35,7 @@ class ResmokeProxyService: return test_list - def read_suite_config(self, suite_name: str) -> Dict[str, Any]: + def read_suite_config(self, suite_name: str) -> dict[str, Any]: """ Read the given resmoke suite configuration. diff --git a/buildscripts/resmoke_tests_runtime_validate.py b/buildscripts/resmoke_tests_runtime_validate.py index c6a021339fc..d1323a76735 100644 --- a/buildscripts/resmoke_tests_runtime_validate.py +++ b/buildscripts/resmoke_tests_runtime_validate.py @@ -5,7 +5,6 @@ import json import sys from collections import namedtuple from statistics import mean -from typing import Dict, List import click import structlog @@ -26,7 +25,7 @@ IGNORE_LESS_THAN_SECS = 15 _TestData = namedtuple("TestData", ["test_file", "duration"]) -def parse_resmoke_report(report_file: str) -> List[TestInfo]: +def parse_resmoke_report(report_file: str) -> list[TestInfo]: """Get js tests info from resmoke report json.""" with open(report_file, "r") as fh: report_data = json.load(fh) @@ -36,7 +35,7 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]: def get_historic_stats( project_id: str, task_name: str, build_variant: str -) -> List[HistoricalTestInformation]: +) -> list[HistoricalTestInformation]: """Get historic test stats.""" base_task_name = get_task_name_without_suffix(task_name, build_variant).replace( BURN_IN_PREFIX, "" @@ -44,7 +43,7 @@ def get_historic_stats( return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant) -def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]: +def make_stats_map(stats: list[_TestData]) -> dict[str, list[float]]: """Make test stats map.""" stats_map = {} diff --git a/buildscripts/resmokelib/bazel_suite_parser.py b/buildscripts/resmokelib/bazel_suite_parser.py index b429c11c07a..f012379abfb 100644 --- a/buildscripts/resmokelib/bazel_suite_parser.py +++ b/buildscripts/resmokelib/bazel_suite_parser.py @@ -11,7 +11,6 @@ subset of Bazel syntax: import functools import os import re -from typing import Dict, List class BazelParseError(Exception): @@ -21,7 +20,7 @@ class BazelParseError(Exception): @functools.cache -def parse_resmoke_suite_test(target_label: str) -> Dict[str, List[str]]: +def parse_resmoke_suite_test(target_label: str) -> dict[str, list[str]]: """Parse a resmoke_suite_test target from BUILD.bazel. Args: target_label: Bazel target label like "//buildscripts/resmokeconfig:core" @@ -121,7 +120,7 @@ def _parse_label(target_label: str) -> tuple[str, str]: return package, target_name -def _parse_load_statements(content: str, package: str) -> Dict[str, str]: +def _parse_load_statements(content: str, package: str) -> dict[str, str]: """Parse load statements from BUILD.bazel content. Extracts identifier to .bzl file mappings from load statements. @@ -172,9 +171,9 @@ def _parse_load_statements(content: str, package: str) -> Dict[str, str]: def _extract_attribute( block: str, attribute_name: str, - identifier_to_bzl_file: Dict[str, str] = None, + identifier_to_bzl_file: dict[str, str] = None, build_file: str = None, -) -> List[str]: +) -> list[str]: """Extract an attribute from a BUILD.bazel rule block. Supports simple lists and list concatenation with identifiers: @@ -232,8 +231,8 @@ def _extract_attribute( def _resolve_identifier_to_labels( - identifier: str, identifier_to_bzl_file: Dict[str, str], build_file: str = None -) -> List[str]: + identifier: str, identifier_to_bzl_file: dict[str, str], build_file: str = None +) -> list[str]: """Convert a Bazel identifier to a list of labels. This function resolves identifiers used in list concatenation expressions. diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index a183a51d6e5..030e21ae8f4 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -16,7 +16,7 @@ import textwrap import traceback from functools import cache from pathlib import Path -from typing import Dict, List, Optional +from typing import Optional import git import pymongo.uri_parser @@ -299,8 +299,8 @@ def _set_up_tracing( otel_collector_dir: Optional[str], trace_id: Optional[str], parent_span_id: Optional[str], - extra_context: Dict[str, object], - suite_files: Optional[List[str]] = None, + extra_context: dict[str, object], + suite_files: Optional[list[str]] = None, shard_index: Optional[int] = None, ) -> bool: """Try to set up otel tracing. On success return True. On failure return False. diff --git a/buildscripts/resmokelib/core/pipe.py b/buildscripts/resmokelib/core/pipe.py index fd00c1e3302..5e363afe0b1 100644 --- a/buildscripts/resmokelib/core/pipe.py +++ b/buildscripts/resmokelib/core/pipe.py @@ -7,7 +7,6 @@ being waited on. import threading from textwrap import wrap -from typing import List # Logkeeper only support log lines up to 4 MB, we want to be a little under that to account for # extra metadata that gets sent along with the log message. @@ -85,7 +84,7 @@ class LoggerPipe(threading.Thread): LoggerPipe.__join(self) # Tidy up the started thread. @staticmethod - def _format_line_for_logging(line_bytes: bytes) -> List[str]: + def _format_line_for_logging(line_bytes: bytes) -> list[str]: """ Convert the given byte array into string(s) to be send to the logger. diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 8796045e0af..7fbffd41429 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -8,7 +8,7 @@ import os import os.path import re import stat -from typing import Any, Optional, Tuple +from typing import Any, Optional from opentelemetry import trace from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator @@ -186,7 +186,7 @@ def mongos_program( executable: Optional[str] = None, process_kwargs: Optional[dict] = None, mongos_options: dict = None, -) -> Tuple[process.Process, dict]: +) -> tuple[process.Process, dict]: """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" bin_version = get_binary_version(executable) args = [executable] @@ -257,7 +257,7 @@ def mongos_program( def mongot_program( logger, job_num, executable=None, process_kwargs=None, mongot_options=None -) -> Tuple[process.Process, Any]: +) -> tuple[process.Process, Any]: """Return a Process instance that starts a mongot.""" args = [executable] mongot_options = mongot_options.copy() diff --git a/buildscripts/resmokelib/discovery/__init__.py b/buildscripts/resmokelib/discovery/__init__.py index 1592be94ccb..50431457ca6 100644 --- a/buildscripts/resmokelib/discovery/__init__.py +++ b/buildscripts/resmokelib/discovery/__init__.py @@ -1,7 +1,7 @@ """Subcommands for test discovery.""" import argparse -from typing import List, Optional +from typing import Optional import yaml from pydantic import BaseModel @@ -25,7 +25,7 @@ class SuiteTestList(BaseModel): """Collection of tests belonging to a suite.""" suite_name: str - tests: List[str] + tests: list[str] class TestDiscoverySubcommand(Subcommand): diff --git a/buildscripts/resmokelib/extensions/add_extensions_signature_pub_key_path.py b/buildscripts/resmokelib/extensions/add_extensions_signature_pub_key_path.py index 9b4aedd7e52..b3282bf916b 100644 --- a/buildscripts/resmokelib/extensions/add_extensions_signature_pub_key_path.py +++ b/buildscripts/resmokelib/extensions/add_extensions_signature_pub_key_path.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from typing import Dict, Optional +from typing import Optional from buildscripts.resmokelib.extensions.constants import ( TEST_PUBLIC_KEY_PATH, @@ -9,8 +9,8 @@ from buildscripts.resmokelib.extensions.constants import ( def add_extensions_signature_pub_key_path( skip_extensions_signature_verification, config, - mongod_options: Dict, - mongos_options: Optional[Dict] = None, + mongod_options: dict, + mongos_options: Optional[dict] = None, ): # We omit providing the extension signature public key path parameter if we intend to skip signature verification. # This signals to the server in insecure mode to skip validating extension signatures at load time. diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 009fc644498..bc0633afe50 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -15,7 +15,7 @@ from abc import ABCMeta, abstractmethod from collections import namedtuple from datetime import datetime, timedelta from io import StringIO -from typing import List, Optional, Tuple +from typing import Optional import psutil from opentelemetry import trace @@ -73,17 +73,17 @@ def get_dumpers(root_logger: logging.Logger, dbg_output: str, include_terminatin return Dumpers(dbg=dbg, jstack=jstack) -def find_files(file_name: str, path: str) -> List[str]: +def find_files(file_name: str, path: str) -> list[str]: matches = glob.glob(f"{path}/**/{file_name}", recursive=True) return [f for f in matches if os.path.isfile(f)] def filter_core_dumps( - core_files: List[str], + core_files: list[str], boring_core_dump_pids: set, max_core_dumps: int, logger: logging.Logger, -) -> List[str]: +) -> list[str]: """ Filter core dump files by removing boring PIDs and applying a maximum cap. @@ -849,7 +849,7 @@ class GDBDumper(Dumper): multiversion_dir: str, logger: logging.Logger, gdb_index_cache: str, - ) -> Tuple[int, str]: # returns (exit_code, test_status) + ) -> tuple[int, str]: # returns (exit_code, test_status) cmds = [] dbg = self.find_debugger() readelf = find_program("eu-readelf", ["/opt/mongodbtoolchain/v5/bin", "/usr/bin"]) diff --git a/buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py b/buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py index ca4153580bf..9951e17448e 100644 --- a/buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py +++ b/buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py @@ -11,7 +11,7 @@ import re import string import sys from abc import ABC, abstractmethod -from typing import List, NamedTuple, Optional +from typing import NamedTuple, Optional from unittest.mock import MagicMock from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskDependency @@ -51,7 +51,7 @@ class CoreAnalysisTaskGenerator(ABC): gdb_index_cache: str, has_interesting_core_dumps: bool, boring_core_dump_pids: set, - ) -> List[FunctionCall]: + ) -> list[FunctionCall]: pass @abstractmethod @@ -216,7 +216,7 @@ class ResmokeCoreAnalysisTaskGenerator(CoreAnalysisTaskGenerator): gdb_index_cache: str, has_interesting_core_dumps: bool, boring_core_dump_pids: set, - ) -> List[FunctionCall]: + ) -> list[FunctionCall]: return _get_core_analyzer_commands( task_id, execution, @@ -279,7 +279,7 @@ class BazelCoreAnalysisTaskGenerator(CoreAnalysisTaskGenerator): gdb_index_cache: str, has_interesting_core_dumps: bool, boring_core_dump_pids: set, - ) -> List[FunctionCall]: + ) -> list[FunctionCall]: return _get_core_analyzer_commands( task_id, execution, @@ -347,7 +347,7 @@ def _get_core_analyzer_commands( has_interesting_core_dumps: bool, boring_core_dump_pids: set, is_bazel_task: bool = False, -) -> List[FunctionCall]: +) -> list[FunctionCall]: """Return setup commands.""" return [ FunctionCall("f_expansions_write"), diff --git a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py index 791973350af..52d19edfda6 100755 --- a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py +++ b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py @@ -18,7 +18,6 @@ import re import signal import sys import traceback -from typing import List import distro import psutil @@ -41,7 +40,7 @@ class HangAnalyzer(Subcommand): """ self.options = options self.root_logger = None - self.interesting_processes: List[str] = [ + self.interesting_processes: list[str] = [ # Remove "python", "java" from the list to avoid hang analyzer multiple invocations "mongo", "mongod", @@ -49,8 +48,8 @@ class HangAnalyzer(Subcommand): "_test", "dbtest", ] - self.go_processes: List[str] = [] - self.process_ids: List[int] = [] + self.go_processes: list[str] = [] + self.process_ids: list[int] = [] def configure_task_id(): run_tid = resmoke_config.EVERGREEN_TASK_ID diff --git a/buildscripts/resmokelib/hang_analyzer/process_list.py b/buildscripts/resmokelib/hang_analyzer/process_list.py index e085a01e334..e84cfdbd14f 100644 --- a/buildscripts/resmokelib/hang_analyzer/process_list.py +++ b/buildscripts/resmokelib/hang_analyzer/process_list.py @@ -4,7 +4,7 @@ import csv import io import os import sys -from typing import List, NamedTuple, Union +from typing import NamedTuple, Union from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program @@ -13,7 +13,7 @@ class Pinfo(NamedTuple): """Holds a vector of PIDs of the same process type.""" name: str - pidv: Union[int, List[int]] + pidv: Union[int, list[int]] def get_processes(process_ids, interesting_processes, process_match, logger): diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 1463d4c2243..5d42ffc24b1 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -1,7 +1,7 @@ """Subcommand for multiversion config.""" import argparse -from typing import List, Optional +from typing import Optional import yaml from pydantic import BaseModel @@ -30,7 +30,7 @@ class MultiversionConfig(BaseModel): * last_continuous_fcv: Continuous version that should be tested against. """ - last_versions: List[str] + last_versions: list[str] requires_fcv_tag: str requires_fcv_tag_lts: str requires_fcv_tag_continuous: str diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index f01bb231b9d..ed0fa35f137 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -4,7 +4,7 @@ from __future__ import annotations import re from bisect import bisect_left, bisect_right -from typing import List, NamedTuple, Optional +from typing import NamedTuple, Optional import structlog import yaml @@ -42,10 +42,10 @@ class VersionConstantValues(NamedTuple): latest: Version last_continuous: Version last_lts: Version - requires_fcv_tag_list: List[Version] - requires_fcv_tag_list_continuous: List[Version] - fcvs_less_than_latest: List[Version] - eols: List[Version] + requires_fcv_tag_list: list[Version] + requires_fcv_tag_list_continuous: list[Version] + fcvs_less_than_latest: list[Version] + eols: list[Version] def get_fcv_tag_list(self) -> str: """Get a comma joined string of all the fcv tags.""" @@ -78,7 +78,7 @@ class VersionConstantValues(NamedTuple): """Get a string version of the latest FCV.""" return version_str(self.latest) - def get_fcv_tags_less_than_latest(self) -> List[str]: + def get_fcv_tags_less_than_latest(self) -> list[str]: """Get the list of all fcv tags less than the latest.""" return [tag_str(fcv) for fcv in self.fcvs_less_than_latest] @@ -102,7 +102,7 @@ class VersionConstantValues(NamedTuple): last_continuous = self.get_last_continuous_fcv() return f"{base_name}-{last_continuous}" - def get_eols(self) -> List[str]: + def get_eols(self) -> list[str]: """Get EOL'd versions as list of strings.""" return [version_str(eol) for eol in self.eols] @@ -148,9 +148,9 @@ class MongoReleases(BaseModel): LTS. """ - feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions") - long_term_support_releases: List[str] = Field(alias="longTermSupportReleases") - eol_versions: List[str] = Field(alias="eolVersions") + feature_compatibility_versions: list[str] = Field(alias="featureCompatibilityVersions") + long_term_support_releases: list[str] = Field(alias="longTermSupportReleases") + eol_versions: list[str] = Field(alias="eolVersions") generate_fcv_lower_bound_override: Optional[str] = Field( None, alias="generateFCVLowerBoundOverride" ) @@ -178,15 +178,15 @@ class MongoReleases(BaseModel): ) raise - def get_fcv_versions(self) -> List[Version]: + def get_fcv_versions(self) -> list[Version]: """Get the Version representation of all fcv versions.""" return [Version(fcv) for fcv in self.feature_compatibility_versions] - def get_lts_versions(self) -> List[Version]: + def get_lts_versions(self) -> list[Version]: """Get the Version representation of the lts versions.""" return [Version(lts) for lts in self.long_term_support_releases] - def get_eol_versions(self) -> List[Version]: + def get_eol_versions(self) -> list[Version]: """Get the Version representation of the EOL versions.""" return [Version(eol) for eol in self.eol_versions] diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index fed9191ad3f..f6bd7a48426 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -14,7 +14,7 @@ import textwrap import time from abc import ABC, abstractmethod from logging import Logger -from typing import List, Optional +from typing import Optional import psutil import yaml @@ -173,7 +173,7 @@ class TestRunner(Subcommand): ) @staticmethod - def _find_suites_by_test(suites: List[Suite]): + def _find_suites_by_test(suites: list[Suite]): """ Look up what other resmoke suites run the tests specified in the suites parameter. @@ -1016,7 +1016,7 @@ class TestRunner(Subcommand): ) suite.tests = config.SHUFFLE_STRATEGY.shuffle(suite.tests) - def _get_suites(self) -> List[Suite]: + def _get_suites(self) -> list[Suite]: """Return the list of suites for this resmoke invocation.""" try: return suitesconfig.get_suites(config.SUITE_FILES, config.TEST_FILES) @@ -1057,11 +1057,11 @@ class TestRunner(Subcommand): @staticmethod def _get_suite_summary(suite: Suite): """Return a summary of the suite run.""" - sb: List[str] = [] + sb: list[str] = [] suite.summarize(sb) return "\n".join(sb) - def _setup_signal_handler(self, suites: List[Suite]): + def _setup_signal_handler(self, suites: list[Suite]): """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump.""" sighandler.register(self._resmoke_logger, suites, self.__start_time) @@ -1166,7 +1166,7 @@ class TestRunnerEvg(TestRunner): return combinations - def _get_suites(self) -> List[Suite]: + def _get_suites(self) -> list[Suite]: """Return a list of resmokelib.testing.suite.Suite instances to execute. For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(), @@ -2549,7 +2549,7 @@ class RunPlugin(PluginInterface): def to_local_args( - input_args: Optional[List[str]] = None, additional_skipped_args: Optional[List[str]] = None + input_args: Optional[list[str]] = None, additional_skipped_args: Optional[list[str]] = None ): """ Return a command line invocation for resmoke.py suitable for being run outside of Evergreen. @@ -2669,7 +2669,7 @@ def to_local_args( ) -def strip_fuzz_config_params(input_args: List[str]): +def strip_fuzz_config_params(input_args: list[str]): """Delete fuzz related command line args because we have to add the seed manually.""" ret = [] diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index b36d6ff0335..adcc2cb038e 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -11,7 +11,7 @@ import os.path import random import subprocess import sys -from typing import Any, List, NamedTuple +from typing import Any, NamedTuple import buildscripts.resmokelib.testing.tags as _tags from buildscripts.resmokelib import config, errors, utils @@ -168,8 +168,8 @@ class TestFileExplorer(object): class _EvaluatePathsResult(NamedTuple): """Results of paths evaluation.""" - evaluated: List[str] - unrecognized: List[str] + evaluated: list[str] + unrecognized: list[str] class _TestList(object): @@ -185,7 +185,7 @@ class _TestList(object): """ def __init__( - self, test_file_explorer: TestFileExplorer, roots: List[str], tests_are_files: bool = True + self, test_file_explorer: TestFileExplorer, roots: list[str], tests_are_files: bool = True ) -> None: """Initialize the _TestList with a TestFileExplorer component and a list of root tests.""" self._test_file_explorer = test_file_explorer @@ -193,7 +193,7 @@ class _TestList(object): self._roots = self._expand_roots(roots) if tests_are_files else roots self._filtered = set(self._roots) - def _evaluate_paths(self, paths: List[str]) -> _EvaluatePathsResult: + def _evaluate_paths(self, paths: list[str]) -> _EvaluatePathsResult: evaluated = [] unrecognized = [] @@ -217,7 +217,7 @@ class _TestList(object): return _EvaluatePathsResult(evaluated, unrecognized) - def _expand_roots(self, tests: List[str]) -> List[str]: + def _expand_roots(self, tests: list[str]) -> list[str]: paths = self._evaluate_paths(tests) if len(paths.unrecognized) > 0 and config.VALIDATE_SELECTOR_PATHS: raise errors.SuiteSelectorConfigurationError( @@ -227,7 +227,7 @@ class _TestList(object): ) return paths.evaluated - def include_files(self, include_files: List[str]) -> None: + def include_files(self, include_files: list[str]) -> None: """ Filter the test list so that it only includes files matching 'include_files'. @@ -259,7 +259,7 @@ class _TestList(object): self._filtered = set(paths.evaluated) - def exclude_files(self, exclude_files: List[str]) -> None: + def exclude_files(self, exclude_files: list[str]) -> None: """ Exclude from the test list the files that match elements from 'exclude_files'. @@ -912,7 +912,7 @@ _SELECTOR_REGISTRY = { def filter_tests( test_kind, selector_config, test_file_explorer=_DEFAULT_TEST_FILE_EXPLORER -) -> tuple[List[str], List[str]]: +) -> tuple[list[str], list[str]]: """Filter the tests according to a specified configuration. Args: @@ -931,7 +931,7 @@ def filter_tests( def group_tests( test_kind, selector_config, test_list, test_file_explorer=_DEFAULT_TEST_FILE_EXPLORER -) -> List[Any]: +) -> list[Any]: """Group the test list according to a specified configuration. Args: diff --git a/buildscripts/resmokelib/setup_multiversion/config.py b/buildscripts/resmokelib/setup_multiversion/config.py index 77bcf80caae..5972c4a3c5e 100644 --- a/buildscripts/resmokelib/setup_multiversion/config.py +++ b/buildscripts/resmokelib/setup_multiversion/config.py @@ -1,7 +1,5 @@ """Setup multiversion config.""" -from typing import List - SETUP_MULTIVERSION_CONFIG = ( "buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml" ) @@ -17,7 +15,7 @@ class Buildvariant: edition: str platform: str architecture: str - versions: List[str] + versions: list[str] def __init__(self, buildvariant_yaml: dict): """Initialize.""" @@ -31,8 +29,8 @@ class Buildvariant: class SetupMultiversionConfig: """Class represents setup multiversion config.""" - evergreen_projects: List[str] - evergreen_buildvariants: List[Buildvariant] + evergreen_projects: list[str] + evergreen_buildvariants: list[Buildvariant] def __init__(self, raw_yaml: dict): """Initialize.""" diff --git a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py index 5a43af55584..c2df41df72e 100644 --- a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py +++ b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py @@ -15,7 +15,7 @@ import subprocess import sys import time from itertools import chain -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, NamedTuple, Optional import distro import yaml @@ -71,7 +71,7 @@ def get_merge_base_commit(version: str, logger: logging.Logger) -> Optional[str] class EvgURLInfo(NamedTuple): """Wrapper around compile URLs with metadata.""" - urls: Dict[str, Any] = {} + urls: dict[str, Any] = {} evg_version_id: str = None @@ -173,7 +173,7 @@ class SetupMultiversion(Subcommand): def _get_release_versions( self, install_last_lts: Optional[bool], install_last_continuous: Optional[bool] - ) -> List[str]: + ) -> list[str]: """Return last-LTS and/or last-continuous versions.""" out = [] if not os.path.isfile( @@ -338,7 +338,7 @@ class SetupMultiversion(Subcommand): "Finished writing binary paths on Windows to %s", config.WINDOWS_BIN_PATHS_FILE ) - def _write_evg_versions_file(self, file_name: str, versions: List[str]): + def _write_evg_versions_file(self, file_name: str, versions: list[str]): with open(file_name, "a") as out: out.write("\n".join(versions)) diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 64c1a0e791b..229dbe23422 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -2,7 +2,7 @@ import threading from logging import Logger -from typing import Generic, List, Optional, TypeVar, Union +from typing import Generic, Optional, TypeVar, Union from opentelemetry import context @@ -71,7 +71,7 @@ class TestSuiteExecutor(object): self._jobs = self._create_jobs(suite.get_num_jobs_to_start()) - def _create_jobs(self, num_jobs: int) -> List[_job.Job]: + def _create_jobs(self, num_jobs: int) -> list[_job.Job]: """ Start jobs. @@ -272,7 +272,7 @@ class TestSuiteExecutor(object): return fixtures.make_fixture(fixture_class, fixture_logger, job_num, **fixture_config) - def _make_hooks(self, fixture, job_num) -> List[Hook]: + def _make_hooks(self, fixture, job_num) -> list[Hook]: """Create the hooks for the job's fixture.""" hooks = [] @@ -376,7 +376,7 @@ class TestQueue(_queue.Queue, Generic[T]): self.max_test_queue_size = utils.default_if_none(_config.MAX_TEST_QUEUE_SIZE, -1) super().__init__() - def add_test_cases(self, test_cases: List[QueueElem]) -> None: + def add_test_cases(self, test_cases: list[QueueElem]) -> None: """Add test cases to the queue.""" for test_case in test_cases: if self.max_test_queue_size < 0 or self.num_tests < self.max_test_queue_size: diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py index fb6a2ade7ed..bbaade70a7c 100644 --- a/buildscripts/resmokelib/testing/fixtures/_builder.py +++ b/buildscripts/resmokelib/testing/fixtures/_builder.py @@ -4,7 +4,7 @@ import json import logging import threading from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Optional from buildscripts.resmokelib import config, errors, logging from buildscripts.resmokelib.suitesconfig import _get_suite_config @@ -143,8 +143,8 @@ class FixtureContainer(object): def _extract_multiversion_options( - kwargs: Dict[str, Any], -) -> Tuple[Optional[List[str]], Optional[str]]: + kwargs: dict[str, Any], +) -> tuple[Optional[list[str]], Optional[str]]: """Pop multiversion options from kwargs dict and return them. :param kwargs: fixture kwargs @@ -176,9 +176,9 @@ class ReplSetBuilder(FixtureBuilder): self, logger: logging.Logger, job_num: int, - fixturelib: Type[FixtureLib], + fixturelib: type[FixtureLib], *args, - existing_nodes: Optional[List[MongoDFixture]] = None, + existing_nodes: Optional[list[MongoDFixture]] = None, **kwargs, ) -> ReplicaSetFixture: """Build a replica set. @@ -269,7 +269,7 @@ class ReplSetBuilder(FixtureBuilder): return replset @staticmethod - def _mutate_kwargs(kwargs: Dict[str, Any]) -> None: + def _mutate_kwargs(kwargs: dict[str, Any]) -> None: """Update replica set fixture kwargs. :param kwargs: replica set fixture kwargs @@ -287,7 +287,7 @@ class ReplSetBuilder(FixtureBuilder): @staticmethod def _validate_multiversion_options( - kwargs: Dict[str, Any], mixed_bin_versions: Optional[List[str]] + kwargs: dict[str, Any], mixed_bin_versions: Optional[list[str]] ) -> None: """Error out if the number of binary versions does not match the number of nodes in replica set. @@ -311,10 +311,10 @@ class ReplSetBuilder(FixtureBuilder): @classmethod def _get_mongod_assets( cls, - kwargs: Dict[str, Any], - mixed_bin_versions: Optional[List[str]], + kwargs: dict[str, Any], + mixed_bin_versions: Optional[list[str]], old_bin_version: Optional[str], - ) -> Tuple[Dict[str, str], Dict[str, str], List[str]]: + ) -> tuple[dict[str, str], dict[str, str], list[str]]: """Make dicts with mongod new/old class and executable names and binary versions. :param kwargs: sharded cluster fixture kwargs @@ -365,7 +365,7 @@ class ReplSetBuilder(FixtureBuilder): def _new_mongod( replset: ReplicaSetFixture, replset_node_index: int, - executables: Dict[str, str], + executables: dict[str, str], _class: str, cur_version: str, is_multiversion: bool, @@ -442,7 +442,7 @@ class ShardedClusterBuilder(FixtureBuilder): LATEST_MONGOS_CLASS = "_MongoSFixture" def build_fixture( - self, logger: logging.Logger, job_num: int, fixturelib: Type[FixtureLib], *args, **kwargs + self, logger: logging.Logger, job_num: int, fixturelib: type[FixtureLib], *args, **kwargs ) -> ShardedClusterFixture: """Build a sharded cluster. @@ -546,7 +546,7 @@ class ShardedClusterBuilder(FixtureBuilder): return sharded_cluster @staticmethod - def _mutate_kwargs(kwargs: Dict[str, Any]) -> None: + def _mutate_kwargs(kwargs: dict[str, Any]) -> None: """Update sharded cluster fixture kwargs. :param kwargs: sharded cluster fixture kwargs @@ -585,7 +585,7 @@ class ShardedClusterBuilder(FixtureBuilder): @staticmethod def _validate_multiversion_options( - kwargs: Dict[str, Any], mixed_bin_versions: Optional[List[str]] + kwargs: dict[str, Any], mixed_bin_versions: Optional[list[str]] ) -> None: """Error out if the number of binary versions does not match the number of nodes in sharded cluster. @@ -606,10 +606,10 @@ class ShardedClusterBuilder(FixtureBuilder): @classmethod def _get_mongos_assets( cls, - kwargs: Dict[str, Any], - mixed_bin_versions: Optional[List[str]], + kwargs: dict[str, Any], + mixed_bin_versions: Optional[list[str]], old_bin_version: Optional[str], - ) -> Tuple[Dict[str, str], Dict[str, str]]: + ) -> tuple[dict[str, str], dict[str, str]]: """Make dicts with mongos new/old class and executable names. :param kwargs: sharded cluster fixture kwargs @@ -670,7 +670,7 @@ class ShardedClusterBuilder(FixtureBuilder): @staticmethod def _new_rs_shard( sharded_cluster: ShardedClusterFixture, - mixed_bin_versions: Optional[List[str]], + mixed_bin_versions: Optional[list[str]], old_bin_version: Optional[str], rs_shard_index: int, num_rs_nodes_per_shard: int, @@ -713,7 +713,7 @@ class ShardedClusterBuilder(FixtureBuilder): @staticmethod def _new_mongos( sharded_cluster: ShardedClusterFixture, - executables: Dict[str, str], + executables: dict[str, str], _class: str, mongos_index: int, total: int, diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py index fa8bbb916e6..ade7f66e412 100644 --- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py +++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py @@ -1,7 +1,6 @@ """Facade wrapping the resmokelib dependencies used by fixtures.""" from logging import Handler, Logger -from typing import Dict from buildscripts.resmokelib import config, core, errors, logging, utils from buildscripts.resmokelib.core import network @@ -121,7 +120,7 @@ class FixtureLib: SET_PARAMETERS_KEY = "set_parameters" - def merge_mongo_option_dicts(self, original: Dict, override: Dict): + def merge_mongo_option_dicts(self, original: dict, override: dict): """ Merge mongod/s options such that --setParameter is merged recursively. diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 7ea9d23b7d3..d69f061a3b3 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -6,7 +6,7 @@ from collections import namedtuple from enum import Enum from logging import Logger from threading import Lock -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import pymongo import pymongo.errors @@ -251,11 +251,11 @@ class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)): def __repr__(self): return "%r(%r, %r)" % (self.__class__.__name__, self.logger, self.job_num) - def get_independent_clusters(self) -> List["Fixture"]: + def get_independent_clusters(self) -> list["Fixture"]: """Return a list of the independent clusters that participate in this fixture.""" return [self] - def get_testable_clusters(self) -> List["Fixture"]: + def get_testable_clusters(self) -> list["Fixture"]: """Return a list of the clusters in this fixture that we want to run tests against.""" return [self] @@ -274,7 +274,7 @@ class _DockerComposeInterface: by leveraging the `all_processes` method to access the startup args. """ - def _all_mongo_d_s_t(self) -> List[Fixture]: + def _all_mongo_d_s_t(self) -> list[Fixture]: """ Return a list of all mongo{d,s,t} `Fixture` instances in this fixture. @@ -284,7 +284,7 @@ class _DockerComposeInterface: "_all_mongo_d_s_t_instances must be implemented by Fixture subclasses that support `docker-compose.yml` generation." ) - def all_processes(self) -> List["Process"]: + def all_processes(self) -> list["Process"]: """ Return a list of all `mongo{d,s,t}` `Process` instances in the fixture. @@ -324,13 +324,13 @@ class MultiClusterFixture(Fixture): REGISTERED_NAME = registry.LEAVE_UNREGISTERED # type: ignore - def get_independent_clusters(self) -> List[Fixture]: + def get_independent_clusters(self) -> list[Fixture]: """Return a list of the independent clusters that participate in this fixture.""" raise NotImplementedError( "get_independent_clusters must be implemented by MultiClusterFixture subclasses" ) - def get_testable_clusters(self) -> List[Fixture]: + def get_testable_clusters(self) -> list[Fixture]: """Return a list of the clusters in this fixture that we want to run tests against.""" return self.get_independent_clusters() @@ -474,7 +474,7 @@ class FixtureTeardownHandler(object): def create_fixture_table(fixture): """Get fixture node info, make it a pretty table. Return it or None if fixture is invalid target.""" - info: List[NodeInfo] = fixture.get_node_info() + info: list[NodeInfo] = fixture.get_node_info() if not info: return None diff --git a/buildscripts/resmokelib/testing/hooks/generate_and_check_perf_results.py b/buildscripts/resmokelib/testing/hooks/generate_and_check_perf_results.py index 8214b341d9d..0d963974ab9 100644 --- a/buildscripts/resmokelib/testing/hooks/generate_and_check_perf_results.py +++ b/buildscripts/resmokelib/testing/hooks/generate_and_check_perf_results.py @@ -6,7 +6,7 @@ import json import subprocess from dataclasses import dataclass from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Optional import requests import tenacity @@ -86,8 +86,8 @@ class GenerateAndCheckPerfResults(interface.Hook): interface.Hook.__init__(self, hook_logger, fixture, GenerateAndCheckPerfResults.DESCRIPTION) self.cedar_report_file = _config.CEDAR_REPORT_FILE self.variant = _config.EVERGREEN_VARIANT_NAME - self.cedar_reports: List[CedarTestReport] = [] - self.performance_thresholds: Dict[str, Any] = {} + self.cedar_reports: list[CedarTestReport] = [] + self.performance_thresholds: dict[str, Any] = {} self.create_time = None self.end_time = None @@ -117,8 +117,8 @@ class GenerateAndCheckPerfResults(interface.Hook): def _check_pass_fail( self, - benchmark_reports: Dict[str, "_BenchmarkThreadsReport"], - cedar_formatted_results: List[CedarTestReport], + benchmark_reports: dict[str, "_BenchmarkThreadsReport"], + cedar_formatted_results: list[CedarTestReport], test, test_report, ): @@ -153,7 +153,7 @@ class GenerateAndCheckPerfResults(interface.Hook): ) continue # Transform the thresholds set into something we can more easily use. - metrics_to_check: List[IndividualMetricThreshold] = [] + metrics_to_check: list[IndividualMetricThreshold] = [] for item in test_thresholds: thread_level = item["thread_level"] for metric in item["metrics"]: @@ -184,7 +184,7 @@ class GenerateAndCheckPerfResults(interface.Hook): ) ) # Transform the reported performance results into something we can more easily use. - transformed_metrics: Dict[ReportedMetric, CedarMetric] = {} + transformed_metrics: dict[ReportedMetric, CedarMetric] = {} for cedar_result in cedar_formatted_results: for individual_metric in cedar_result.metrics: reported_metric = ReportedMetric( @@ -244,8 +244,8 @@ class GenerateAndCheckPerfResults(interface.Hook): ) def _generate_cedar_report( - self, benchmark_reports: Dict[str, "_BenchmarkThreadsReport"] - ) -> List[CedarTestReport]: + self, benchmark_reports: dict[str, "_BenchmarkThreadsReport"] + ) -> list[CedarTestReport]: """Format the data to look like a cedar report.""" cedar_report = [] @@ -264,10 +264,10 @@ class GenerateAndCheckPerfResults(interface.Hook): return cedar_report - def _parse_report(self, report_dict) -> Dict[str, "_BenchmarkThreadsReport"]: + def _parse_report(self, report_dict) -> dict[str, "_BenchmarkThreadsReport"]: context = report_dict["context"] # Reports grouped by name without thread. - benchmark_reports: Dict[str, "_BenchmarkThreadsReport"] = {} + benchmark_reports: dict[str, "_BenchmarkThreadsReport"] = {} for benchmark_res in report_dict["benchmarks"]: bm_name_obj = _BenchmarkThreadsReport.parse_bm_name(benchmark_res) @@ -284,7 +284,7 @@ class GenerateAndCheckPerfResults(interface.Hook): task_name: str, variant: str, measurement: str, - args: Dict[str, Any], + args: dict[str, Any], base_commit: str, project: str, ) -> Optional[int]: @@ -360,12 +360,12 @@ class CheckPerfResultTestCase(interface.DynamicTestCase): description, base_test_name, hook, - thresholds_to_check: List["IndividualMetricThreshold"], - reported_metrics: Dict[ReportedMetric, CedarMetric], + thresholds_to_check: list["IndividualMetricThreshold"], + reported_metrics: dict[ReportedMetric, CedarMetric], ): super().__init__(logger, test_name, description, base_test_name, hook) - self.thresholds_to_check: List["IndividualMetricThreshold"] = thresholds_to_check - self.reported_metrics: Dict[ReportedMetric, CedarMetric] = reported_metrics + self.thresholds_to_check: list["IndividualMetricThreshold"] = thresholds_to_check + self.reported_metrics: dict[ReportedMetric, CedarMetric] = reported_metrics self.github: Github = Github(get_expansion("github_token_mongo")) def run_test(self): @@ -577,7 +577,7 @@ class _BenchmarkThreadsReport(object): """Add to report.""" self.thread_benchmark_map[bm_name_obj.thread_count].append(report) - def generate_cedar_metrics(self) -> Dict[int, List[CedarMetric]]: + def generate_cedar_metrics(self) -> dict[int, list[CedarMetric]]: """ Generate metrics for Cedar. @@ -714,7 +714,7 @@ class _BenchmarkThreadsReport(object): return res @staticmethod - def check_dup_metric_names(metrics: List[CedarMetric]) -> bool: + def check_dup_metric_names(metrics: list[CedarMetric]) -> bool: """Check duplicated metric names for Cedar.""" names = [] for metric in metrics: @@ -724,7 +724,7 @@ class _BenchmarkThreadsReport(object): return False @staticmethod - def parse_bm_name(benchmark_res: Dict[str, Any]): + def parse_bm_name(benchmark_res: dict[str, Any]): """ Split the benchmark name into base_name, thread_count and statistic_type. diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 9753c399d77..a413bb102b0 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -5,7 +5,7 @@ import sys import threading import time from collections import namedtuple -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from opentelemetry import context, trace from opentelemetry.context.context import Context @@ -41,7 +41,7 @@ class Job(object): job_num: int, logger: logging.Logger, fixture: Fixture, - hooks: List[Hook], + hooks: list[Hook], report: TestReport, archival: HookTestArchival, suite_options: config.SuiteOptions, diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index e730ff78c70..a5d045912a6 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -7,7 +7,7 @@ import threading import time from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor, TimeoutError -from typing import Any, Dict, List, Optional +from typing import Any, Optional from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import selector as _selector @@ -123,7 +123,7 @@ class Suite(object): self._tests, self._excluded = self._get_tests_for_kind(self.test_kind) return self._excluded - def _get_tests_for_kind(self, test_kind) -> tuple[List[any], List[str]]: + def _get_tests_for_kind(self, test_kind) -> tuple[list[any], list[str]]: """Return the tests to run and those that were excluded, based on the 'test_kind'-specific filtering policy.""" selector_config = self.get_selector_config() @@ -376,7 +376,7 @@ class Suite(object): return self._reports @synchronized - def summarize(self, sb: List[str]): + def summarize(self, sb: list[str]): """Append a summary of the suite onto the string builder 'sb'.""" if not self._reports and not self._partial_reports: sb.append("No tests ran.") @@ -585,7 +585,7 @@ class Suite(object): RETURN_STATUS = "suite_return_status" ERRORNO = "suite_errorno" - def get_suite_otel_attributes(self) -> Dict[str, Any]: + def get_suite_otel_attributes(self) -> dict[str, Any]: attributes = { Suite.METRIC_NAMES.DISPLAY_NAME: self.get_display_name(), Suite.METRIC_NAMES.NAME: self.get_name(), @@ -602,8 +602,8 @@ class Suite(object): @staticmethod def filter_tests_for_shard( - tests: List[str], shard_count: Optional[int], shard_index: Optional[int] - ) -> List[str]: + tests: list[str], shard_count: Optional[int], shard_index: Optional[int] + ) -> list[str]: """Filter tests to only include those that should be run by this shard.""" if shard_index is None or shard_count is None: return tests @@ -625,15 +625,15 @@ class Suite(object): class ShardingStrategy(ABC): @abstractmethod def get_tests_for_shard( - self, tests: List[str], shard_count: int, shard_index: int - ) -> List[str]: + self, tests: list[str], shard_count: int, shard_index: int + ) -> list[str]: pass class EqualTestCount(ShardingStrategy): def get_tests_for_shard( - self, tests: List[str], shard_count: int, shard_index: int - ) -> List[str]: + self, tests: list[str], shard_count: int, shard_index: int + ) -> list[str]: return [test_case for i, test_case in enumerate(tests) if i % shard_count == shard_index] @@ -644,8 +644,8 @@ class EqualRuntime(ShardingStrategy): self.runtimes[runtime["test_name"]] = runtime["avg_duration_pass"] def get_tests_for_shard( - self, tests: List[str], shard_count: int, shard_index: int - ) -> List[str]: + self, tests: list[str], shard_count: int, shard_index: int + ) -> list[str]: shards = [[] for _ in range(shard_count)] shard_runtimes = [0] * shard_count diff --git a/buildscripts/resmokelib/testing/symbolizer_service.py b/buildscripts/resmokelib/testing/symbolizer_service.py index e5ae3c9171c..82f4cd28f71 100644 --- a/buildscripts/resmokelib/testing/symbolizer_service.py +++ b/buildscripts/resmokelib/testing/symbolizer_service.py @@ -10,7 +10,7 @@ import sys import time from datetime import timedelta from threading import Lock -from typing import Any, List, NamedTuple, Optional, Set +from typing import Any, NamedTuple, Optional from opentelemetry import trace @@ -224,7 +224,7 @@ class ResmokeSymbolizer: def symbolize_stacktraces( self, test: TestCase, - files: List[str], + files: list[str], symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS, ) -> None: """ @@ -259,7 +259,7 @@ class ResmokeSymbolizer: def create_unsymbolized_instructions( self, test: TestCase, - unsymbolized_stacktraces_info: List[dict], + unsymbolized_stacktraces_info: list[dict], expansions_file: str = "../expansions.yml", ): """ @@ -409,7 +409,7 @@ If no symbolized stacktrace is created, then most likely either: return dbpath - def collect_stacktrace_files(self, dir_path: str) -> List[str]: + def collect_stacktrace_files(self, dir_path: str) -> list[str]: """ Collect all stacktrace files which are not empty and return their full paths. @@ -434,7 +434,7 @@ class FileService: self._processed_files = self.load_processed_files(processed_files_list_path) @staticmethod - def load_processed_files(file_path: str) -> Set[str]: + def load_processed_files(file_path: str) -> set[str]: """ Load processed files info from a file. @@ -446,7 +446,7 @@ class FileService: return {line for line in set(file.readlines()) if line} return set() - def add_to_processed_files(self, files: List[str]) -> None: + def add_to_processed_files(self, files: list[str]) -> None: """ Bulk add to collection of processed files. @@ -477,7 +477,7 @@ class FileService: return file in self._processed_files @staticmethod - def find_all_children_recursively(dir_path: str) -> List[str]: + def find_all_children_recursively(dir_path: str) -> list[str]: """ Find all children files in directory recursively. @@ -490,7 +490,7 @@ class FileService: return children_in_dir @staticmethod - def filter_by_extension(files: List[str], extension: str) -> List[str]: + def filter_by_extension(files: list[str], extension: str) -> list[str]: """ Filter files by extension. @@ -501,7 +501,7 @@ class FileService: return [f for f in files if f.endswith(extension)] @staticmethod - def filter_out_non_files(files: List[str]) -> List[str]: + def filter_out_non_files(files: list[str]) -> list[str]: """ Filter out non files. @@ -510,7 +510,7 @@ class FileService: """ return [f for f in files if os.path.isfile(f)] - def filter_out_already_processed_files(self, files: List[str]): + def filter_out_already_processed_files(self, files: list[str]): """ Filter out already processed files. @@ -520,7 +520,7 @@ class FileService: return [f for f in files if not self.is_processed(f)] @staticmethod - def filter_out_empty_files(files: List[str]) -> List[str]: + def filter_out_empty_files(files: list[str]) -> list[str]: """ Filter our files that are empty. diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py index d71f9d90217..7f945e102d0 100644 --- a/buildscripts/resmokelib/testing/testcases/interface.py +++ b/buildscripts/resmokelib/testing/testcases/interface.py @@ -10,7 +10,7 @@ import threading import timeit import unittest import uuid -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Optional import psutil @@ -19,7 +19,7 @@ from buildscripts.resmokelib.hang_analyzer.hang_analyzer import HangAnalyzer from buildscripts.resmokelib.utils import registry from buildscripts.resmokelib.utils.self_test_fakes import test_analysis -_TEST_CASES: Dict[str, Callable] = {} # type: ignore +_TEST_CASES: dict[str, Callable] = {} # type: ignore def make_test_case(test_kind, *args, **kwargs) -> "TestCase": @@ -138,7 +138,7 @@ class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TE DYNAMIC = "test_dynamic" BACKGROUND = "test_background" - def get_test_otel_attributes(self) -> Dict[str, Any]: + def get_test_otel_attributes(self) -> dict[str, Any]: return { TestCase.METRIC_NAMES.BASE_NAME: self.basename(), TestCase.METRIC_NAMES.LONG_NAME: self.long_name(), diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py index a4a9cd5b8b5..993d60ffe78 100644 --- a/buildscripts/resmokelib/utils/archival.py +++ b/buildscripts/resmokelib/utils/archival.py @@ -12,7 +12,6 @@ import threading import time from abc import ABC, abstractmethod from pathlib import Path -from typing import List from buildscripts.resmokelib import config @@ -107,13 +106,13 @@ def get_archive_path(path: str) -> str: class ArchiveStrategy(ABC): @abstractmethod - def archive_files(self, files: List[str], archive_name: str, display_name: str): + def archive_files(self, files: list[str], archive_name: str, display_name: str): pass def exit(self): pass - def create_tar_archive(self, files: List[str], archive: Path, display_name: str): + def create_tar_archive(self, files: list[str], archive: Path, display_name: str): status = 0 size_mb = 0 message = "Tar/gzip {} files: {}".format(display_name, files) @@ -284,7 +283,7 @@ class ArchiveToS3(ArchiveStrategy): self._archive_file_worker.join(timeout=timeout) self.check_thread(self._archive_file_worker, False) - def archive_files(self, files: List[str], archive_name: str, display_name: str): + def archive_files(self, files: list[str], archive_name: str, display_name: str): _, temp_file = tempfile.mkstemp(suffix=".tgz") status, message, size_mb = self.create_tar_archive(files, temp_file, display_name) self._upload_queue.put( @@ -307,7 +306,7 @@ class ArchiveToDirectory(ArchiveStrategy): self.logger = logger os.makedirs(self.directory, exist_ok=True) - def archive_files(self, files: List[str], archive_name: str, display_name: str): + def archive_files(self, files: list[str], archive_name: str, display_name: str): archive = os.path.join(self.directory, archive_name) with open(archive, "w") as _: pass # Touch create the archive file, so we can check the size of the disk it is on before adding to it. @@ -319,7 +318,7 @@ class TestArchival(ArchiveStrategy): self.archive_file = os.path.join(config.DBPATH_PREFIX, "test_archival.txt") self.logger = logger - def archive_files(self, files: List[str], archive_name: str, display_name: str): + def archive_files(self, files: list[str], archive_name: str, display_name: str): self.logger.info(f"These files/directories would have been archived: {files}") with open(self.archive_file, "a") as f: for file in files: @@ -367,7 +366,7 @@ class Archival(object): # Lock to control access from multiple threads. self._lock = threading.Lock() - def archive_files(self, input_files: str | List[str], archive_name: str, display_name: str): + def archive_files(self, input_files: str | list[str], archive_name: str, display_name: str): if isinstance(input_files, str): input_files = [input_files] diff --git a/buildscripts/resmokelib/utils/evergreen_conn.py b/buildscripts/resmokelib/utils/evergreen_conn.py index 39fa72ed077..0c6c1b5fd24 100644 --- a/buildscripts/resmokelib/utils/evergreen_conn.py +++ b/buildscripts/resmokelib/utils/evergreen_conn.py @@ -4,7 +4,7 @@ import os import pathlib from collections import deque from pathlib import Path -from typing import Deque, Iterator, List, Optional, Set, Union +from typing import Iterator, Optional, Union import requests import structlog @@ -34,9 +34,9 @@ class EvergreenConnError(Exception): pass -def _find_evergreen_yaml_candidates() -> List[str]: +def _find_evergreen_yaml_candidates() -> list[str]: # Common for machines in Evergreen - candidates: List[Union[str, Path]] = [os.getcwd()] + candidates: list[Union[str, Path]] = [os.getcwd()] cwd = pathlib.Path(os.getcwd()) # add every path that is the parent of CWD as well @@ -199,7 +199,7 @@ def get_compile_artifact_urls( evg_build = evg_api.build_by_id(build_id) LOGGER.debug("Found evergreen build.", evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") - evg_tasks: Deque[Union[Task, str]] = deque(evg_build.get_tasks()) + evg_tasks: deque[Union[Task, str]] = deque(evg_build.get_tasks()) tasks_wrapper = _filter_successful_tasks(evg_api, evg_tasks) LOGGER.info( "Found the following multiversion tasks", @@ -273,7 +273,7 @@ def _get_multiversion_urls(tasks_wrapper: _MultiversionTasks): def _filter_successful_tasks( - evg_api: RetryingEvergreenApi, evg_tasks: Deque[Union[Task, str]] + evg_api: RetryingEvergreenApi, evg_tasks: deque[Union[Task, str]] ) -> _MultiversionTasks: """ We want to filter successful tasks in order by variant then by dependent tasks to find the compile tasks. @@ -283,7 +283,7 @@ def _filter_successful_tasks( compile_task = None archive_symbols_task = None push_task = None - seen_task_ids: Set[str] = set() + seen_task_ids: set[str] = set() while evg_tasks: evg_task = evg_tasks.popleft() diff --git a/buildscripts/resmokelib/utils/otel_id_generator.py b/buildscripts/resmokelib/utils/otel_id_generator.py index 1e201714cb5..fde66ae64bc 100644 --- a/buildscripts/resmokelib/utils/otel_id_generator.py +++ b/buildscripts/resmokelib/utils/otel_id_generator.py @@ -1,6 +1,6 @@ import random import time -from typing import List, Optional +from typing import Optional from opentelemetry import trace from opentelemetry.sdk.trace import IdGenerator @@ -19,7 +19,7 @@ class ResmokeOtelIdGenerator(IdGenerator): run in parallel with the same traceID and parentSpanID. """ - def __init__(self, suite_files: Optional[List[str]] = None, shard_index: Optional[int] = None): + def __init__(self, suite_files: Optional[list[str]] = None, shard_index: Optional[int] = None): """ Initialize the unique span ID generator. diff --git a/buildscripts/sbom_linter.py b/buildscripts/sbom_linter.py index 376009116b9..891283b62f3 100644 --- a/buildscripts/sbom_linter.py +++ b/buildscripts/sbom_linter.py @@ -2,7 +2,6 @@ import argparse import json import os import sys -from typing import List import jsonschema from license_expression import get_spdx_licensing @@ -43,7 +42,7 @@ class ErrorManager: def __init__(self, input_file: str): self.input_file: str = input_file self.component_name: str = "" - self.errors: List[str] = [] + self.errors: list[str] = [] def update_component_attribute(self, component_name: str) -> None: self.component_name = component_name diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py index 16908d812ae..48700370fcd 100644 --- a/buildscripts/simple_report.py +++ b/buildscripts/simple_report.py @@ -3,7 +3,6 @@ import json import os import pathlib -from typing import List import click from typing_extensions import TypedDict @@ -25,7 +24,7 @@ class Report(TypedDict): """Evergreen report.""" failures: int - results: List[Result] + results: list[Result] def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str: @@ -68,7 +67,7 @@ def try_combine_reports(out: Report): pass -def _dedup_lines(lines: List[str]) -> List[str]: +def _dedup_lines(lines: list[str]) -> list[str]: return list(set(lines)) diff --git a/buildscripts/smoke_tests/smoke_tests.py b/buildscripts/smoke_tests/smoke_tests.py index d3431b780ba..4bda8a79200 100755 --- a/buildscripts/smoke_tests/smoke_tests.py +++ b/buildscripts/smoke_tests/smoke_tests.py @@ -24,7 +24,7 @@ from collections import defaultdict, deque from dataclasses import dataclass from pathlib import Path from socket import gethostname -from typing import Any, Deque, Dict, List, Optional, Set, Union +from typing import Any, Optional, Union from rich.status import Status @@ -62,10 +62,10 @@ from buildscripts.resmokelib.utils.evergreen_conn import get_evergreen_api @dataclass class Node: name: str - args: List[str] - popen_kwargs: Dict[str, str] + args: list[str] + popen_kwargs: dict[str, str] log_file: Path - deps: Set["Node"] + deps: set["Node"] _start_time: Optional[float] = None _finish_time: Optional[float] = None _proc: Optional[subprocess.Popen] = None @@ -104,11 +104,11 @@ class Node: self._finish_time = time.monotonic() return self._proc.returncode - def deps_are_satisfied(self, finished: Set["Node"]): + def deps_are_satisfied(self, finished: set["Node"]): return self.deps.issubset(finished) -def normalize_deps(x: Union[None, Node, Set[Node]]): +def normalize_deps(x: Union[None, Node, set[Node]]): if x is None: return set() elif isinstance(x, (tuple, list, set)): @@ -117,7 +117,7 @@ def normalize_deps(x: Union[None, Node, Set[Node]]): return {x} -def send_slack_notification(nodes: List[Node], total_elapsed: float, component_name: str): +def send_slack_notification(nodes: list[Node], total_elapsed: float, component_name: str): overall_success = True lines = [ "```", @@ -176,11 +176,11 @@ class CommandRunner: ): self._log_path = log_path self._parallelism = parallelism - self._downstream: Dict[Node, Set[Node]] = defaultdict(set) - self._nodes: Set[Node] = set() - self._finished: Set[Node] = set() - self._ready: Deque[Node] = deque() - self._running: Set[Node] = set() + self._downstream: dict[Node, set[Node]] = defaultdict(set) + self._nodes: set[Node] = set() + self._finished: set[Node] = set() + self._ready: deque[Node] = deque() + self._running: set[Node] = set() self._status = Status(status=f"{component_name} smoke tests") self._start_time = time.monotonic() self._finish_time: Optional[float] = None @@ -205,9 +205,9 @@ class CommandRunner: self, *, name: str, - args: List[Any], + args: list[Any], log_file: str, - deps: Union[None, Node, Set[Node]] = None, + deps: Union[None, Node, set[Node]] = None, **kwargs, ) -> Node: log_file = self._log_path.joinpath(log_file) @@ -234,7 +234,7 @@ class CommandRunner: print(f"Logging results to {self._log_path}") self._status.start() try: - iter_finished: Set[Node] = set() + iter_finished: set[Node] = set() while self._finished != self._nodes: while len(self._running) < self._parallelism and len(self._ready) > 0: node = self._ready.popleft() @@ -327,7 +327,7 @@ def run_smoke_tests( *, component_name, log_path: Path, - bazel_args: List[str], + bazel_args: list[str], run_clang_tidy: bool, send_slack_notification: bool, ): diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 4e65529f7ed..ad1fe284ff5 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -12,7 +12,6 @@ import sys import time import unittest from shutil import rmtree -from typing import List import yaml @@ -674,7 +673,7 @@ class TestDiscovery(_ResmokeSelftest): ) -def execute_resmoke(resmoke_args: List[str], subcommand: str = "run"): +def execute_resmoke(resmoke_args: list[str], subcommand: str = "run"): return subprocess.run( [sys.executable, "buildscripts/resmoke.py", subcommand] + resmoke_args, text=True, diff --git a/buildscripts/tests/resmokelib/testing/hooks/test_generate_and_check_perf_results.py b/buildscripts/tests/resmokelib/testing/hooks/test_generate_and_check_perf_results.py index 1359258a3e6..b3c8d6ae6c4 100755 --- a/buildscripts/tests/resmokelib/testing/hooks/test_generate_and_check_perf_results.py +++ b/buildscripts/tests/resmokelib/testing/hooks/test_generate_and_check_perf_results.py @@ -4,7 +4,6 @@ import datetime import logging import unittest -from typing import Dict, List import mock @@ -373,7 +372,7 @@ class TestBenchmarkThreadsReport(GenerateAndCheckPerfResultsFixture): class TestCheckPerfResultTestCase(unittest.TestCase): def test_all_metrics_pass(self): - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -383,7 +382,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=1) @@ -402,7 +401,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): test_case.run_test() def test_a_metric_fails(self): - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -412,7 +411,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -457,7 +456,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): mock_repo.get_pull.return_value = mock_pr mock_github.get_repo.return_value = mock_repo - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -467,7 +466,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -513,7 +512,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): mock_repo.get_pull.return_value = mock_pr mock_github.get_repo.return_value = mock_repo - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -523,7 +522,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -569,7 +568,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): mock_repo.get_pull.return_value = mock_pr mock_github.get_repo.return_value = mock_repo - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -579,7 +578,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -625,7 +624,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): mock_repo.get_pull.return_value = mock_pr mock_github.get_repo.return_value = mock_repo - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -635,7 +634,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -682,7 +681,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): mock_repo.get_pull.return_value = mock_pr mock_github.get_repo.return_value = mock_repo - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -692,7 +691,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -723,7 +722,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): "github_token_mongo": "fake_token", }.get(key, default) - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -733,7 +732,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -763,7 +762,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): "github_token_mongo": "fake_token", }.get(key, default) - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -773,7 +772,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=100) @@ -793,7 +792,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): test_case.run_test() def test_metric_doesnt_exist(self): - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -803,7 +802,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=1, metric_name="instructions" ): CedarMetric(name="instructions", type="LATENCY", value=1) @@ -822,7 +821,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): test_case.run_test() def test_thread_level_doesnt_exist(self): - thresholds_to_check: List[cbr.IndividualMetricThreshold] = [ + thresholds_to_check: list[cbr.IndividualMetricThreshold] = [ cbr.IndividualMetricThreshold( metric_name="latency", thread_level=1, @@ -832,7 +831,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase): threshold_limit=20, ) ] - reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = { + reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = { cbr.ReportedMetric( test_name="fake-test", thread_level=12, metric_name="latency" ): CedarMetric(name="latency", type="LATENCY", value=1) diff --git a/buildscripts/timeouts/timeout_service.py b/buildscripts/timeouts/timeout_service.py index 41ad6ba956b..418e01d87a0 100644 --- a/buildscripts/timeouts/timeout_service.py +++ b/buildscripts/timeouts/timeout_service.py @@ -1,6 +1,6 @@ """Service for determining task timeouts.""" -from typing import Any, Dict, NamedTuple, Optional +from typing import Any, NamedTuple, Optional import inject import structlog @@ -209,7 +209,7 @@ class TimeoutService: return clean_every_n_cadence - def _get_hook_config(self, suite_name: str, hook_name: str) -> Optional[Dict[str, Any]]: + def _get_hook_config(self, suite_name: str, hook_name: str) -> Optional[dict[str, Any]]: """ Get the configuration for the given hook. diff --git a/buildscripts/todo_check.py b/buildscripts/todo_check.py index a4629d9ce6a..d27d1a2f989 100755 --- a/buildscripts/todo_check.py +++ b/buildscripts/todo_check.py @@ -6,7 +6,7 @@ import re import sys from collections import defaultdict from dataclasses import dataclass -from typing import Callable, Dict, Iterable, List, NamedTuple, Optional +from typing import Callable, Iterable, NamedTuple, Optional import click @@ -70,9 +70,9 @@ class FoundTodos: by_file: All the references found mapped by the files they were found in. """ - no_tickets: List[Todo] - with_tickets: Dict[str, List[Todo]] - by_file: Dict[str, List[Todo]] + no_tickets: list[Todo] + with_tickets: dict[str, list[Todo]] + by_file: dict[str, list[Todo]] class TodoChecker: @@ -111,7 +111,7 @@ class TodoChecker: walk_fs(base_dir, self.check_file) @staticmethod - def print_todo_list(todo_list: List[Todo]) -> None: + def print_todo_list(todo_list: list[Todo]) -> None: """Display all the TODOs in the given list.""" last_file = None for todo in todo_list: diff --git a/buildscripts/tracing_profiler/profilerlib.py b/buildscripts/tracing_profiler/profilerlib.py index 802fa6ae6ec..990976aa50b 100644 --- a/buildscripts/tracing_profiler/profilerlib.py +++ b/buildscripts/tracing_profiler/profilerlib.py @@ -1,6 +1,5 @@ import json from dataclasses import dataclass -from typing import Dict @dataclass @@ -12,7 +11,7 @@ class SpanMetrics: net_nanos: int exclusive_nanos: int count: int - children: Dict[str, int] + children: dict[str, int] def to_dict(self): return { @@ -28,7 +27,7 @@ class SpanMetrics: @dataclass class CallMetrics: - spans: Dict[int, SpanMetrics] + spans: dict[int, SpanMetrics] @staticmethod def new_empty(): diff --git a/buildscripts/util/buildozer_utils.py b/buildscripts/util/buildozer_utils.py index 95ac6fd3245..47b9a970dc8 100644 --- a/buildscripts/util/buildozer_utils.py +++ b/buildscripts/util/buildozer_utils.py @@ -1,8 +1,7 @@ import subprocess -from typing import List -def _bd_command(cmd: str, labels: List[str]): +def _bd_command(cmd: str, labels: list[str]): print( f"buildozer '{cmd}' " + " ".join(labels), ) @@ -19,15 +18,15 @@ def _bd_command(cmd: str, labels: List[str]): return p -def bd_add(labels: List[str], attr: str, values: List[str]) -> None: +def bd_add(labels: list[str], attr: str, values: list[str]) -> None: _bd_command(f'add {attr} {" ".join(values)}', labels) -def bd_set(labels: List[str], attr: str, value: str) -> None: +def bd_set(labels: list[str], attr: str, value: str) -> None: _bd_command(f"set {attr} {value}", labels) -def bd_remove(labels: List[str], attr: str, values: List[str]) -> None: +def bd_remove(labels: list[str], attr: str, values: list[str]) -> None: _bd_command(f'remove {attr} {" ".join(values)}', labels) @@ -35,26 +34,26 @@ def bd_new(package: str, rule_kind: str, rule_name: str) -> None: _bd_command(f"new {rule_kind} {rule_name}", [package]) -def bd_comment(labels: List[str], comment: str, attr: str = "", value: str = "") -> None: +def bd_comment(labels: list[str], comment: str, attr: str = "", value: str = "") -> None: _bd_command(f"comment {attr} {value} {comment}", labels) -def bd_print(labels: List[str], attrs: List[str]) -> str: +def bd_print(labels: list[str], attrs: list[str]) -> str: p = _bd_command(f'print {" ".join(attrs)}', labels) return p.stdout -def bd_new_load(packages: List[str], path: str, rules: List[str]) -> None: +def bd_new_load(packages: list[str], path: str, rules: list[str]) -> None: _bd_command(f'new_load {path} {" ".join(rules)}', packages) -def bd_fix(fixes: List[str]) -> None: +def bd_fix(fixes: list[str]) -> None: _bd_command(f'fix {" ".join(fixes)}', []) -def bd_copy(labels: List[str], attr: str, from_rule: str) -> None: +def bd_copy(labels: list[str], attr: str, from_rule: str) -> None: _bd_command(f"copy {attr} {from_rule}", labels) -def bd_move(labels: List[str], old_attr: str, new_attr: str) -> None: +def bd_move(labels: list[str], old_attr: str, new_attr: str) -> None: _bd_command(f"move {old_attr} {new_attr} *", labels) diff --git a/buildscripts/util/cedar_report.py b/buildscripts/util/cedar_report.py index 548aea9579f..5c7a1fa22c9 100644 --- a/buildscripts/util/cedar_report.py +++ b/buildscripts/util/cedar_report.py @@ -1,7 +1,7 @@ """Cedar report.""" from dataclasses import dataclass -from typing import List, Union +from typing import Union @dataclass @@ -29,7 +29,7 @@ class CedarTestReport: test_name: str thread_level: int - metrics: List[CedarMetric] + metrics: list[CedarMetric] def as_dict(self) -> dict: """Return dictionary representation.""" diff --git a/buildscripts/util/codeowners_utils.py b/buildscripts/util/codeowners_utils.py index 46ddbf2d4ba..a2f62b7713c 100644 --- a/buildscripts/util/codeowners_utils.py +++ b/buildscripts/util/codeowners_utils.py @@ -2,13 +2,12 @@ import fnmatch import os import re from functools import lru_cache -from typing import Dict, List, Tuple import yaml @lru_cache -def process_owners(cur_dir: str) -> Tuple[Dict[re.Pattern, List[str]], bool]: +def process_owners(cur_dir: str) -> tuple[dict[re.Pattern, list[str]], bool]: if not cur_dir: return ({}, False) @@ -54,7 +53,7 @@ class Owners: open("buildscripts/util/co_jira_map.yml", "r", encoding="utf8") ) - def get_codeowners(self, file_path: str) -> List[str]: + def get_codeowners(self, file_path: str) -> list[str]: cur_dir = os.path.dirname(file_path) codeowners = [] # search up tree until matching filter found @@ -71,10 +70,10 @@ class Owners: cur_dir = os.path.dirname(cur_dir) return codeowners - def get_jira_team_from_codeowner(self, codeowner: str) -> List[str]: + def get_jira_team_from_codeowner(self, codeowner: str) -> list[str]: return self.co_jira_map[codeowner] - def get_jira_team_owner(self, file_path: str) -> List[str]: + def get_jira_team_owner(self, file_path: str) -> list[str]: return [ jira_team for codeowner in self.get_codeowners(file_path) diff --git a/buildscripts/util/fileops.py b/buildscripts/util/fileops.py index 4b9f7292368..f7948e09128 100644 --- a/buildscripts/util/fileops.py +++ b/buildscripts/util/fileops.py @@ -1,7 +1,7 @@ """Utility to support file operations.""" import os -from typing import Any, Dict +from typing import Any import yaml @@ -63,7 +63,7 @@ def write_file_to_dir(directory: str, file: str, contents: str, overwrite: bool write_file(target_file, contents) -def read_yaml_file(path: str) -> Dict[str, Any]: +def read_yaml_file(path: str) -> dict[str, Any]: """ Read the yaml file at the given path and return the contents. diff --git a/buildscripts/util/oauth.py b/buildscripts/util/oauth.py index fa212931da5..bc7e209f4bf 100644 --- a/buildscripts/util/oauth.py +++ b/buildscripts/util/oauth.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from http.server import BaseHTTPRequestHandler, HTTPServer from random import choice from string import ascii_lowercase -from typing import Any, Callable, Optional, Tuple +from typing import Any, Callable, Optional from urllib.parse import parse_qs, urlsplit from webbrowser import open as web_open @@ -117,7 +117,7 @@ class _RedirectServer(HTTPServer): def __init__( self, - server_address: Tuple[str, int], + server_address: tuple[str, int], handler: Callable[..., BaseHTTPRequestHandler], redirect_uri: str, auth_domain: str, diff --git a/buildscripts/util/teststats.py b/buildscripts/util/teststats.py index 6906ce9418b..38b219a35d0 100644 --- a/buildscripts/util/teststats.py +++ b/buildscripts/util/teststats.py @@ -4,7 +4,7 @@ from collections import defaultdict from dataclasses import dataclass from itertools import chain from json import JSONDecodeError -from typing import Callable, List, NamedTuple, Optional +from typing import Callable, NamedTuple, Optional import requests from requests.adapters import HTTPAdapter, Retry @@ -126,11 +126,11 @@ class HistoricTestInfo(NamedTuple): test_name: str num_pass: int avg_duration: float - hooks: List[HistoricHookInfo] + hooks: list[HistoricHookInfo] @classmethod def from_test_stats( - cls, test_stats: HistoricalTestInformation, hooks: List[HistoricHookInfo] + cls, test_stats: HistoricalTestInformation, hooks: list[HistoricHookInfo] ) -> "HistoricTestInfo": """Create an instance from a test_stats object.""" return cls( @@ -176,12 +176,12 @@ class HistoricTestInfo(NamedTuple): class HistoricTaskData(object): """Represent the test statistics for the task that is being analyzed.""" - def __init__(self, historic_test_results: List[HistoricTestInfo]) -> None: + def __init__(self, historic_test_results: list[HistoricTestInfo]) -> None: """Initialize the TestStats with raw results from the Evergreen API.""" self.historic_test_results = historic_test_results @staticmethod - def get_stats_from_s3(project: str, task: str, variant: str) -> List[HistoricalTestInformation]: + def get_stats_from_s3(project: str, task: str, variant: str) -> list[HistoricalTestInformation]: """ Retrieve test stats from s3 for a given task. @@ -217,7 +217,7 @@ class HistoricTaskData(object): @classmethod def from_stats_list( - cls, historical_test_data: List[HistoricalTestInformation] + cls, historical_test_data: list[HistoricalTestInformation] ) -> "HistoricTaskData": """ Build historic task data from a list of historic stats. @@ -240,7 +240,7 @@ class HistoricTaskData(object): ] ) - def get_tests_runtimes(self) -> List[TestRuntime]: + def get_tests_runtimes(self) -> list[TestRuntime]: """Return the list of (test_file, runtime_in_secs) tuples ordered by decreasing runtime.""" tests = [ TestRuntime( diff --git a/evergreen/functions/binaries_extract.py b/evergreen/functions/binaries_extract.py index 8aa4f0c882c..0aee0cf21fe 100644 --- a/evergreen/functions/binaries_extract.py +++ b/evergreen/functions/binaries_extract.py @@ -29,12 +29,11 @@ import pathlib import shutil import subprocess import sys -from typing import List ZSTD_EXTRACTION = "tar --zstd -xf" -def get_cmd(tarball: str, extraction_command: str) -> List[str]: +def get_cmd(tarball: str, extraction_command: str) -> list[str]: shell = os.environ.get("SHELL", "/bin/bash") if sys.platform == "win32": proc = subprocess.run( diff --git a/evergreen/validate_compile_commands.py b/evergreen/validate_compile_commands.py index ffda110c483..42eb5bde6e7 100644 --- a/evergreen/validate_compile_commands.py +++ b/evergreen/validate_compile_commands.py @@ -9,7 +9,7 @@ import shlex import subprocess import sys import tempfile -from typing import Any, Dict, Iterator, List, Tuple +from typing import Any, Iterator default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY") if not default_dir: @@ -48,7 +48,7 @@ def _parse_repo_env_from_bazelrc(bazelrc_path: str, var_name: str) -> str | None return None -def _capture_msvc_env(vs_vc_dir: str, arch: str) -> Dict[str, str]: +def _capture_msvc_env(vs_vc_dir: str, arch: str) -> dict[str, str]: """Run vcvarsall.bat and capture the environment it sets.""" # Some environments may include surrounding quotes in BAZEL_VC/BAZEL_VS. vs_vc_dir = vs_vc_dir.strip().strip('"').strip("'") @@ -63,13 +63,13 @@ def _capture_msvc_env(vs_vc_dir: str, arch: str) -> Dict[str, str]: vcvarsall = os.path.normpath(vcvarsall).strip().strip('"').strip("'") - def _run_cmd_capture_env(cmd: List[str]) -> Dict[str, str]: + def _run_cmd_capture_env(cmd: list[str]) -> dict[str, str]: proc = subprocess.run(cmd, capture_output=True, text=True) if proc.returncode != 0: raise RuntimeError( f"Failed to run vcvarsall.bat (rc={proc.returncode}). stderr:\n{proc.stderr}" ) - env: Dict[str, str] = {} + env: dict[str, str] = {} for line in proc.stdout.splitlines(): if "=" not in line: continue @@ -102,7 +102,7 @@ def _capture_msvc_env(vs_vc_dir: str, arch: str) -> Dict[str, str]: pass -def _maybe_add_windows_toolchain_env(base_env: Dict[str, str], repo_root: str) -> Dict[str, str]: +def _maybe_add_windows_toolchain_env(base_env: dict[str, str], repo_root: str) -> dict[str, str]: """On Windows, ensure INCLUDE/LIB/PATH are set by loading a VS developer env.""" if platform.system() != "Windows": return base_env @@ -135,7 +135,7 @@ def _maybe_add_windows_toolchain_env(base_env: Dict[str, str], repo_root: str) - return merged -def _iter_compiledb_entries(path: str) -> Iterator[Dict[str, Any]]: +def _iter_compiledb_entries(path: str) -> Iterator[dict[str, Any]]: """Stream parse compile_commands.json (a JSON array of objects) without loading it all in memory.""" decoder = json.JSONDecoder() buf = "" @@ -203,7 +203,7 @@ def _hash_file_name(file_name: str) -> int: return int.from_bytes(digest[:8], byteorder="big", signed=False) -def _make_test_compile_args(args: List[str]) -> List[str]: +def _make_test_compile_args(args: list[str]) -> list[str]: """Convert a compile command into a 'test compile' command. By default we keep the compilation semantics (not syntax-only), but we can optionally @@ -253,7 +253,7 @@ def _map_writable_output_path(out_root: str, original_path: str) -> str: return comp drive, tail = os.path.splitdrive(original_path) - parts: List[str] = [] + parts: list[str] = [] if drive: # Drive may be "C:" or a UNC prefix like "\\\\server\\share". @@ -277,8 +277,8 @@ def _map_writable_output_path(out_root: str, original_path: str) -> str: def _rewrite_output_paths_to_writable_dir( - args: List[str], cwd: str, out_root: str, entry_output: str | None = None -) -> List[str]: + args: list[str], cwd: str, out_root: str, entry_output: str | None = None +) -> list[str]: """Rewrite output-producing args (-o, -MF, /Fo, etc.) into a writable directory.""" if not args: return args @@ -291,7 +291,7 @@ def _rewrite_output_paths_to_writable_dir( return os.path.normcase(os.path.normpath(abs_p)) # Collect all plausible output paths (compile_commands "output" can differ from actual /Fo). - orig_outs: List[str] = [] + orig_outs: list[str] = [] if isinstance(entry_output, str) and entry_output: orig_outs.append(entry_output) @@ -313,8 +313,8 @@ def _rewrite_output_paths_to_writable_dir( i += 1 # Build mapping by normalized absolute path to destination. - out_map: Dict[str, str] = {} - dep_map: Dict[str, str] = {} + out_map: dict[str, str] = {} + dep_map: dict[str, str] = {} for o in orig_outs: if not o: continue @@ -401,7 +401,7 @@ def _rewrite_output_paths_to_writable_dir( return rewritten -def _ensure_parent_dirs_exist_for_outputs(args: List[str], cwd: str, repo_root: str) -> None: +def _ensure_parent_dirs_exist_for_outputs(args: list[str], cwd: str, repo_root: str) -> None: """Create parent dirs for any output/deps paths referenced by the command. This is intentionally conservative: it only creates directories for paths that resolve @@ -460,12 +460,12 @@ def _ensure_parent_dirs_exist_for_outputs(args: List[str], cwd: str, repo_root: i += 1 -def _select_entries_for_test_compile(path: str, n: int) -> Tuple[int, List[Dict[str, Any]]]: +def _select_entries_for_test_compile(path: str, n: int) -> tuple[int, list[dict[str, Any]]]: """Pick N entries by sorting deterministic hashes of entry['file'] and taking the first N.""" # Keep a max-heap of the N smallest hashes. # IMPORTANT: include stable, comparable tie-breakers so heapq never compares dicts. # Tuple: (-hash, file_name, seq, entry) - heap: List[Tuple[int, str, int, Dict[str, Any]]] = [] + heap: list[tuple[int, str, int, dict[str, Any]]] = [] total = 0 seq = 0 for entry in _iter_compiledb_entries(path): @@ -510,7 +510,7 @@ def main() -> int: ) os.makedirs(out_root, exist_ok=True) - def _prep_entry(entry: Dict[str, Any]) -> Tuple[str, str, List[str]]: + def _prep_entry(entry: dict[str, Any]) -> tuple[str, str, list[str]]: args = entry.get("arguments") if args is None and isinstance(entry.get("command"), str): # Fallback for standard compile_commands format. @@ -535,7 +535,7 @@ def main() -> int: ) return (file_name, directory, test_args) - work: List[Tuple[str, str, List[str]]] = [] + work: list[tuple[str, str, list[str]]] = [] for entry in selected: file_name, directory, test_args = _prep_entry(entry) if file_name and directory and test_args: @@ -552,7 +552,7 @@ def main() -> int: print(f"Running {len(work)} test compiles...", flush=True) compile_env = _maybe_add_windows_toolchain_env(os.environ.copy(), repo_root=default_dir) - def _run_one(item: Tuple[str, str, List[str]]) -> Tuple[str, int, List[str], str, str]: + def _run_one(item: tuple[str, str, list[str]]) -> tuple[str, int, list[str], str, str]: file_name, directory, test_args = item _ensure_parent_dirs_exist_for_outputs(test_args, cwd=directory, repo_root=default_dir) proc = subprocess.run( diff --git a/jstests/ssl/x509/mkcert.py b/jstests/ssl/x509/mkcert.py index 926f5069aad..b4bb48dadf0 100755 --- a/jstests/ssl/x509/mkcert.py +++ b/jstests/ssl/x509/mkcert.py @@ -14,7 +14,7 @@ import re import shutil import subprocess import tempfile -from typing import Any, Dict +from typing import Any import mkdigest import OpenSSL @@ -36,7 +36,7 @@ except: CONFIGFILE = "jstests/ssl/x509/certs.yml" -CONFIG = Dict[str, Any] +CONFIG = dict[str, Any] # tlsfeature = status_request isn't supported by older versions of OpenSSL so we manually define this below # 1.3.6.1.5.5.7.1.24: "tls_feature" extension as defined in https://tools.ietf.org/html/rfc7633#section-6 diff --git a/modules_poc/private_headers.py b/modules_poc/private_headers.py index 4636bcb1e26..14391cfafd6 100755 --- a/modules_poc/private_headers.py +++ b/modules_poc/private_headers.py @@ -3,17 +3,15 @@ import os import sys from pathlib import Path -from itertools import chain from codeowners import path_to_regex from collections.abc import Callable from re import Pattern import typer -import glob sys.path.append(os.path.dirname(os.path.abspath(__file__))) from browse import load_decls, File, is_submodule_usage -from mod_mapping import mod_for_file, teams_for_file, glob_paths, normpath_for_file +from mod_mapping import mod_for_file, teams_for_file, normpath_for_file from merge_decls import get_file_family_regex REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") diff --git a/pyproject.toml b/pyproject.toml index 81b302c17ce..cdefcbaeef3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -217,8 +217,9 @@ target-version = "py310" [tool.ruff.lint] extend-select = [ - "I", # isort - "W605", # invalid-escape-sequence + "I", # isort + "UP006", # non-pep585-annotation + "W605", # invalid-escape-sequence ] ignore = [ "E402", # module-import-not-at-top-of-file diff --git a/src/mongo/db/query/query_tester/scripts/utils.py b/src/mongo/db/query/query_tester/scripts/utils.py index e8931fc9287..e8539bb4a3d 100644 --- a/src/mongo/db/query/query_tester/scripts/utils.py +++ b/src/mongo/db/query/query_tester/scripts/utils.py @@ -13,7 +13,7 @@ import argparse import os import sys from pathlib import Path -from typing import Optional, Tuple +from typing import Optional MONGODB_URI = "mongodb://127.0.0.1:27017/" @@ -27,7 +27,7 @@ def clean_up_file(file_path: str, desc: str) -> None: print_and_exit(f"Error: {desc} {file_path} does not exist for cleanup.") -def construct_filenames(output_prefix: str, suffix: Optional[str] = None) -> Tuple[str, str]: +def construct_filenames(output_prefix: str, suffix: Optional[str] = None) -> tuple[str, str]: """Constructs standardized filenames based on the prefix and optional suffix.""" pkl_file = f"{output_prefix}.pkl" json_file = f"{output_prefix}_{suffix}.json" if suffix else f"{output_prefix}.json" @@ -77,7 +77,7 @@ def print_and_exit(msg: str) -> None: sys.exit(1) -def validate_and_change_directory(feature_extractor_dir: str) -> Tuple[Path, Path]: +def validate_and_change_directory(feature_extractor_dir: str) -> tuple[Path, Path]: """Validates and changes to feature-extractor directory.""" # Validate feature-extractor directory and change to it diff --git a/src/mongo/db/query/query_tester/tests/scripts/setup_repro_env.py b/src/mongo/db/query/query_tester/tests/scripts/setup_repro_env.py index 67da367826f..6caedb2091b 100644 --- a/src/mongo/db/query/query_tester/tests/scripts/setup_repro_env.py +++ b/src/mongo/db/query/query_tester/tests/scripts/setup_repro_env.py @@ -12,7 +12,6 @@ import re import subprocess import sys from pathlib import Path -from typing import Dict, List REPO_NAME_PREFIX = "query-correctness-tests-" @@ -79,7 +78,7 @@ def extract_repo_and_subpath(test_path: Path): ) -def process_log_file(logfile: Path, failing_tests_map: Dict[str, Dict[str, List[int]]]): +def process_log_file(logfile: Path, failing_tests_map: dict[str, dict[str, list[int]]]): """ Parses evergreen log files by regex matching relevant parts of the query_tester output to determine the failing test files, and the corresponding test numbers. @@ -132,7 +131,7 @@ def create_repo_commit_map(test_repos_conf_file: Path): return repo_commit_map -def fetch_test_repos_dirs(failing_tests_map, repo_commit_map: Dict[str, str], clone_dir: Path): +def fetch_test_repos_dirs(failing_tests_map, repo_commit_map: dict[str, str], clone_dir: Path): """ Fetch the query-correctness-tests-* repos with the failing tests, checks out the repos to the correct commit hash and only checks out the failing test files. @@ -191,7 +190,7 @@ def fetch_test_repos_dirs(failing_tests_map, repo_commit_map: Dict[str, str], cl subprocess.run(command) -def build_query_tester_command(failing_tests_map: Dict[str, Dict[str, List[int]]], out_dir: Path): +def build_query_tester_command(failing_tests_map: dict[str, dict[str, list[int]]], out_dir: Path): """ Find all the failing test files from the failing_tests_map and output a formatted string that can be used as an argument to the mongotest binary diff --git a/src/mongo/mongo_config_header.py b/src/mongo/mongo_config_header.py index 533f638cf45..05e43d8820a 100644 --- a/src/mongo/mongo_config_header.py +++ b/src/mongo/mongo_config_header.py @@ -6,7 +6,6 @@ import platform import subprocess import tempfile import threading -from typing import Dict logfile_path: str = "" loglock = threading.Lock() @@ -442,7 +441,7 @@ def get_config_header_substs(): def generate_config_header( compiler_path, compiler_args, env_vars, logpath, additional_inputs=[], extra_definitions={} -) -> Dict[str, str]: +) -> dict[str, str]: global logfile_path CompilerSettings.compiler_path = compiler_path CompilerSettings.compiler_args = compiler_args diff --git a/src/mongo/util/version_constants_gen.py b/src/mongo/util/version_constants_gen.py index 8d097b2b2b4..5640a9ef568 100644 --- a/src/mongo/util/version_constants_gen.py +++ b/src/mongo/util/version_constants_gen.py @@ -6,7 +6,6 @@ import platform import re import subprocess import threading -from typing import Dict # This function gets the running OS as identified by Python @@ -196,7 +195,7 @@ def log_check(message): def generate_config_header( compiler_path, compiler_args, env_vars, logpath, additional_inputs, extra_definitions={} -) -> Dict[str, str]: +) -> dict[str, str]: global logfile_path logfile_path = logpath diff --git a/x509/mkcert.py b/x509/mkcert.py index 601bc134ae2..e276ccef7a9 100644 --- a/x509/mkcert.py +++ b/x509/mkcert.py @@ -5,7 +5,7 @@ import hashlib import ipaddress import json from pathlib import PurePath -from typing import Any, Dict +from typing import Any import asn1crypto.core as asn1 import cryptography.hazmat.primitives.serialization.pkcs12 as pkcs12 @@ -37,7 +37,7 @@ OID_ORDER = [NAME_TO_OID[n].dotted_string for n in ["C", "ST", "L", "O", "OU", " CONFIGFILE = None # Config parsed as YAML. -CONFIG = Dict[str, Any] +CONFIG = dict[str, Any] # <= 825 in order to abide by https://support.apple.com/en-us/HT210176. MAX_VALIDITY_PERIOD_DAYS = 824