mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-119906: Enable UP006 ruff rule (#48284)

Browse Source 
GitOrigin-RevId: 2069fa7bda111a89d4a9a43a56e71f06cc2e9a7c
This commit is contained in:
Nick Jefferies 2026-02-19 15:57:12 -05:00 committed by MongoDB Bot
parent c172464fdf
commit a4130ea9ab
113 changed files with 640 additions and 681 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
bazel/auto_header/auto_header.py
View File

@ -1,13 +1,11 @@
import subprocess
import hashlib
import os
import threading
import time
import json
from collections import defaultdict
from functools import lru_cache
from pathlib import Path
from typing import Dict, List, Set, Tuple
from io import StringIO
@ -201,8 +199,8 @@ def _is_third_party_path(p: str) -> bool:
)
def _build_left_rights_map(grouped: Dict[str, Dict[str, Set[str]]]) -> Dict[str, Set[str]]:
out: Dict[str, Set[str]] = {}
def _build_left_rights_map(grouped: dict[str, dict[str, set[str]]]) -> dict[str, set[str]]:
out: dict[str, set[str]] = {}
for d, files in grouped.items():
for f, rights in files.items():
out[f"{d}/{f}"] = set(r for r in rights if r.startswith("mongo/"))
@ -221,7 +219,7 @@ def _repo_to_file_label(repo_rel: str) -> str:
def _dfs_flatten_from_seed(
seed_repo_path: str, lr_map: Dict[str, Set[str]], visited: Set[str], out_labels: Set[str]
seed_repo_path: str, lr_map: dict[str, set[str]], visited: set[str], out_labels: set[str]
) -> None:
srp = seed_repo_path.replace("\\", "/")
if _is_third_party_path(srp) or _is_excluded_right(srp):
@ -335,15 +333,15 @@ def _cleanup_from_manifest(prev_dirs: set[str], curr_dirs: set[str]) -> int:
return removed
def _compute_flat_idl_from_lr_map(lr_map: Dict[str, Set[str]], seeds: List[str]) -> Set[str]:
out: Set[str] = set()
vis: Set[str] = set()
def _compute_flat_idl_from_lr_map(lr_map: dict[str, set[str]], seeds: list[str]) -> set[str]:
out: set[str] = set()
vis: set[str] = set()
for seed in seeds:
_dfs_flatten_from_seed(seed, lr_map, vis, out)
return {lab for lab in out if "third_party" not in lab.split("/")}
def _inject_flat_group(grouped: Dict[str, Dict[str, Set[str]]], flat_labels: Set[str]) -> None:
def _inject_flat_group(grouped: dict[str, dict[str, set[str]]], flat_labels: set[str]) -> None:
# Store as a synthetic "left" whose rights are already absolute labels.
# We'll special-case its rendering to dump labels verbatim.
grouped.setdefault("mongo", {}).setdefault("idl_headers_flat", set()).update(flat_labels)
@ -366,7 +364,7 @@ def augment_with_idl_placeholders(
def augment_with_generated_left(
grouped: Dict[str, Dict[str, Set[str]]], gen_map: Dict[str, List[str]]
grouped: dict[str, dict[str, set[str]]], gen_map: dict[str, list[str]]
) -> None:
"""
For each generated left entry, ensure a filegroup exists and add the
@ -388,7 +386,7 @@ def augment_with_generated_left(
def augment_with_source_placeholders(
grouped: Dict[str, Dict[str, Set[str]]], src_paths: List[str]
grouped: dict[str, dict[str, set[str]]], src_paths: list[str]
) -> None:
"""Seed a filegroup for every left source/header even if it has 0 includes."""
for p in src_paths:
@ -548,7 +546,7 @@ package(default_visibility = [{vis_list}])
"""
def _dir_digest(src_dir: str, files_map: Dict[str, Set[str]], *, visibility: str) -> str:
def _dir_digest(src_dir: str, files_map: dict[str, set[str]], *, visibility: str) -> str:
lines = []
lines.append(f"{VERSION_SALT}|{visibility}|{src_dir}\n")
for left in sorted(files_map):
@ -573,7 +571,7 @@ def _read_existing_digest(path: Path) -> str:
def _build_content_for_dir(
src_dir: str, file_to_rights: Dict[str, Set[str]], *, visibility: list[str], digest: str
src_dir: str, file_to_rights: dict[str, set[str]], *, visibility: list[str], digest: str
) -> str:
pkg = f"{SRC_ROOT_POSIX}/{src_dir}/{AUTO_DIR}"
buf = StringIO()
@ -589,8 +587,8 @@ def _build_content_for_dir(
write("filegroup(\n")
write(f' name = "{emit_name}",\n')
seen: Set[str] = set()
labels: List[str] = []
seen: set[str] = set()
labels: list[str] = []
if not is_flat:
for own in _owning_labels_for_left(src_dir, left_file):
@ -623,7 +621,7 @@ def _build_content_for_dir(
return buf.getvalue()
def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: Dict[str, Set[str]]) -> str | None:
def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: dict[str, set[str]]) -> str | None:
if not gen_header_repo.endswith("_gen.h"):
return None
base = gen_header_repo[: -len("_gen.h")]
@ -637,8 +635,8 @@ def _guess_idl_from_gen_header(gen_header_repo: str, lr_map: Dict[str, Set[str]]
def _write_build_for_group(
src_dir: str, files_map: Dict[str, Set[str]], *, fsync_write: bool
) -> Tuple[str, bool]:
src_dir: str, files_map: dict[str, set[str]], *, fsync_write: bool
) -> tuple[str, bool]:
out_dir = SRC_ROOT / src_dir / AUTO_DIR
out_path = out_dir / BUILD_NAME
visibility = module_visibility_for_build_dir(src_dir)
@ -736,7 +734,7 @@ def collect_left_right_pairs(rg_bin: str) -> list[tuple[str, str]]:
return list(merged)
def parse_left_right_pairs(pairs: list[tuple[str, str]]) -> Dict[str, Dict[str, Set[str]]]:
def parse_left_right_pairs(pairs: list[tuple[str, str]]) -> dict[str, dict[str, set[str]]]:
by_srcdir = defaultdict(lambda: defaultdict(set))
add = set.add
src_exts = SRC_EXTS_TUPLE
@ -759,7 +757,7 @@ def _env_bool(var: str, default: bool) -> bool:
return v not in ("0", "false", "False", "no", "No", "")
def gen_auto_headers(repo_root: Path) -> Dict[str, object]:
def gen_auto_headers(repo_root: Path) -> dict[str, object]:
"""
Runs the header generation once and returns a result dict:
{ ok: bool, err: str|None, wrote: int, skipped: int, t_ms: float }

3
bazel/config/generate_config_header.py
View File

@ -10,10 +10,9 @@ import inspect
import os
import sys
import textwrap
from typing import Dict
def write_config_header(input_path: str, output_path: str, definitions: Dict[str, str]) -> None:
def write_config_header(input_path: str, output_path: str, definitions: dict[str, str]) -> None:
with open(input_path) as in_file:
content = in_file.read()

10
bazel/format/rules_lint_format_wrapper.py
View File

@ -2,7 +2,7 @@ import argparse
import os
import pathlib
import subprocess
from typing import List, Union
from typing import Union
from git import Repo
from utils.evergreen_git import get_mongodb_remote
@ -45,7 +45,7 @@ def _git_unstaged_files() -> str:
return result.stdout.strip() + os.linesep
def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]:
def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> list[str]:
"""Query git to get a list of files in the repo from a diff."""
# There are 3 diffs we run:
# 1. List of commits between origin/master and HEAD of current branch
@ -72,7 +72,7 @@ def run_rules_lint(
rules_lint_format_path: pathlib.Path,
rules_lint_format_check_path: pathlib.Path,
check: bool,
files_to_format: Union[List[str], str] = "all",
files_to_format: Union[list[str], str] = "all",
) -> bool:
try:
if check:
@ -91,7 +91,7 @@ def run_rules_lint(
def run_prettier(
prettier: pathlib.Path, check: bool, files_to_format: Union[List[str], str] = "all"
prettier: pathlib.Path, check: bool, files_to_format: Union[list[str], str] = "all"
) -> bool:
# Explicitly ignore anything in the output directories or any symlinks in the root of the repository
# to prevent bad symlinks from failing the run, see https://github.com/prettier/prettier/issues/11568 as
@ -218,7 +218,7 @@ def main() -> int:
else:
files_to_format = _get_files_changed_since_fork_point(origin_branch)
def files_to_format_contains_bazel_file(files: Union[List[str], str]) -> bool:
def files_to_format_contains_bazel_file(files: Union[list[str], str]) -> bool:
if files == "all":
return True
return any(file.endswith(".bazel") or "BUILD" in file for file in files)

8
bazel/gpg_signer.py
View File

@ -18,7 +18,7 @@ import shutil
import subprocess
import sys
import tempfile
from typing import List, Optional
from typing import Optional
debug = False # manually change to enable verbose output
@ -28,7 +28,7 @@ def _debug(msg: str) -> None:
print(msg, file=sys.stderr)
def _run(argv: List[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess:
def _run(argv: list[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess:
if capture_stdout:
return subprocess.run(
argv, check=True, text=True, stdout=subprocess.PIPE, stderr=sys.stderr
@ -51,7 +51,7 @@ def _extract_fingerprint(colons_output: str) -> Optional[str]:
return None
def main(argv: List[str]) -> int:
def main(argv: list[str]) -> int:
_debug("Starting gpg_signer.py")
if len(argv) != 6:
@ -106,7 +106,7 @@ def main(argv: List[str]) -> int:
return 1
# Build passphrase options if provided.
pass_opts: List[str] = []
pass_opts: list[str] = []
if passphrase_file:
pass_opts = ["--pinentry-mode", "loopback", "--passphrase-file", passphrase_file]

20
bazel/merge_tidy_configs.py
View File

@ -13,14 +13,14 @@ from __future__ import annotations
import argparse
import pathlib
from typing import Any, Dict, List
from typing import Any
import yaml
# --------------------------- YAML helpers ---------------------------
def load_yaml(file_path: str | pathlib.Path) -> Dict[str, Any]:
def load_yaml(file_path: str | pathlib.Path) -> dict[str, Any]:
path = pathlib.Path(file_path)
if not path.exists():
raise SystemExit(f"Error: Config file '{file_path}' not found.")
@ -28,11 +28,11 @@ def load_yaml(file_path: str | pathlib.Path) -> Dict[str, Any]:
return yaml.safe_load(f) or {}
def split_checks_to_list(value: Any) -> List[str]:
def split_checks_to_list(value: Any) -> list[str]:
"""Normalize a Checks value (string or list) into a flat list of tokens."""
if value is None:
return []
parts: List[str] = []
parts: list[str] = []
if isinstance(value, str):
parts = [s.strip() for s in value.split(",")]
elif isinstance(value, list):
@ -42,7 +42,7 @@ def split_checks_to_list(value: Any) -> List[str]:
def merge_checks_into_config(
target_config: Dict[str, Any], incoming_config: Dict[str, Any]
target_config: dict[str, Any], incoming_config: dict[str, Any]
) -> None:
"""Append incoming Checks onto target Checks (string-concatenated)."""
accumulated = split_checks_to_list(target_config.get("Checks"))
@ -51,9 +51,9 @@ def merge_checks_into_config(
target_config["Checks"] = ",".join(accumulated + additions)
def check_options_list_to_map(value: Any) -> Dict[str, Any]:
def check_options_list_to_map(value: Any) -> dict[str, Any]:
"""Transform CheckOptions list[{key,value}] into a dict[key]=value."""
out: Dict[str, Any] = {}
out: dict[str, Any] = {}
if isinstance(value, list):
for item in value:
if isinstance(item, dict) and "key" in item:
@ -62,7 +62,7 @@ def check_options_list_to_map(value: Any) -> Dict[str, Any]:
def merge_check_options_into_config(
target_config: Dict[str, Any], incoming_config: Dict[str, Any]
target_config: dict[str, Any], incoming_config: dict[str, Any]
) -> None:
"""
Merge CheckOptions so later configs override earlier by 'key'.
@ -161,9 +161,9 @@ def main() -> None:
parser.add_argument("--out", required=True, help="Output merged YAML path.")
args = parser.parse_args()
merged_config: Dict[str, Any] = load_yaml(args.baseline)
merged_config: dict[str, Any] = load_yaml(args.baseline)
config_paths: List[pathlib.Path] = filter_and_sort_config_paths(
config_paths: list[pathlib.Path] = filter_and_sort_config_paths(
args.config_files, args.scope_dir
)

8
bazel/mongot_extension_signing_key/gpg_export_armored_key.py
View File

@ -14,7 +14,7 @@ import shutil
import subprocess
import sys
import tempfile
from typing import List, Optional
from typing import Optional
debug = False # manually change to enable verbose output
@ -24,7 +24,7 @@ def _debug(msg: str) -> None:
print(msg, file=sys.stderr)
def _run(argv: List[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess:
def _run(argv: list[str], *, capture_stdout: bool = False) -> subprocess.CompletedProcess:
if capture_stdout:
return subprocess.run(
argv, check=True, text=True, stdout=subprocess.PIPE, stderr=sys.stderr
@ -47,7 +47,7 @@ def _extract_fingerprint(colons_output: str) -> Optional[str]:
return None
def main(argv: List[str]) -> int:
def main(argv: list[str]) -> int:
_debug("Starting gpg_export_armored_key.py")
if len(argv) != 5:
@ -104,7 +104,7 @@ def main(argv: List[str]) -> int:
_debug("gpg-agent extracted fingerprint: " + fpr)
# Build passphrase options if provided.
pass_opts: List[str] = []
pass_opts: list[str] = []
if passphrase_file:
pass_opts = ["--pinentry-mode", "loopback", "--passphrase-file", passphrase_file]

1
bazel/symbol_checker/symbol_checker.py
View File

@ -1,7 +1,6 @@
import sys
import argparse
import json
import os
parser = argparse.ArgumentParser()
parser.add_argument("--sym", required=True)

3
bazel/wrapper_hook/flag_sync.py
View File

@ -3,7 +3,6 @@ import pathlib
import sys
import time
import traceback
from typing import Dict
REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
sys.path.append(str(REPO_ROOT))
@ -23,7 +22,7 @@ ALLOW_LINES = [
]
def update_bazelrc(flags: Dict[str, Dict], verbose: bool):
def update_bazelrc(flags: dict[str, dict], verbose: bool):
bazelrc_path = f"{REPO_ROOT}/.bazelrc.sync"
if verbose:
print(f"Updating {bazelrc_path}")

16
bazel/wrapper_hook/lint.py
View File

@ -6,7 +6,7 @@ import shutil
import subprocess
import sys
import tempfile
from typing import List, Optional
from typing import Optional
REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
sys.path.append(str(REPO_ROOT))
@ -82,7 +82,7 @@ all_subpackage_javascript_files()
print(f"Created BUILD.bazel in {full_dir}")
def list_files_with_targets(bazel_bin: str) -> List:
def list_files_with_targets(bazel_bin: str) -> list:
return [
line.strip()
for line in subprocess.run(
@ -102,10 +102,10 @@ class LintRunner:
def list_files_without_targets(
self,
files_with_targets: List[str],
files_with_targets: list[str],
type_name: str,
ext: str,
dirs: List[str],
dirs: list[str],
) -> bool:
# rules_lint only checks files that are in targets, verify that all files in the source tree
# are contained within targets.
@ -173,14 +173,14 @@ class LintRunner:
else:
print(f"All {type_name} files have BUILD.bazel targets!")
def run_bazel(self, target: str, args: List = []):
def run_bazel(self, target: str, args: list = []):
p = subprocess.run([self.bazel_bin, "run", target] + (["--"] + args if args else []))
if p.returncode != 0:
self.fail = True
if not self.keep_going:
raise LinterFail("Linter failed")
def simple_file_size_check(self, files_to_lint: List[str]):
def simple_file_size_check(self, files_to_lint: list[str]):
for file in files_to_lint:
if os.path.getsize(file) > LARGE_FILE_THRESHOLD:
print(f"File {file} exceeds large file threshold of {LARGE_FILE_THRESHOLD}")
@ -302,7 +302,7 @@ def _git_unstaged_files() -> str:
return result.stdout.strip() + os.linesep
def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]:
def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> list[str]:
"""Query git to get a list of files in the repo from a diff."""
# There are 3 diffs we run:
# 1. List of commits between origin/master and HEAD of current branch
@ -379,7 +379,7 @@ def lint_mod(lint_runner: LintRunner):
# subprocess.run([bazel_bin, "run", "//modules_poc:browse", "--", "merged_decls.json", "--parse-only"], check=True)
def run_rules_lint(bazel_bin: str, args: List[str]):
def run_rules_lint(bazel_bin: str, args: list[str]):
parsed_args, args = get_parsed_args(args)
if platform.system() == "Windows":
print("eslint not supported on windows")

6
buildscripts/bazel_burn_in.py
View File

@ -27,7 +27,7 @@ import re
import subprocess
import sys
from functools import cache
from typing import List, NamedTuple
from typing import NamedTuple
import typer
import yaml
@ -141,7 +141,7 @@ def get_resmoke_configs():
return yaml.safe_load(f)
def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]:
def query_targets_to_burn_in(origin_rev: str) -> list[BurnInTargetInfo]:
change_detector = LocalFileChangeDetector(origin_rev)
tests_changed = change_detector.find_changed_tests([Repo(".")])
@ -183,7 +183,7 @@ def query_targets_to_burn_in(origin_rev: str) -> List[BurnInTargetInfo]:
@cache
def get_targets_with_tag(tag: str) -> List[str]:
def get_targets_with_tag(tag: str) -> list[str]:
try:
query = f"attr(tags, '\\b{tag}(?![a-zA-Z0-9_-])', //...)"
result = subprocess.run(

3
buildscripts/bazel_custom_formatter.py
View File

@ -7,7 +7,6 @@ import sys
import time
from collections import deque
from pathlib import Path
from typing import Dict, List
sys.path.append(".")
@ -54,7 +53,7 @@ def find_group(unittest_paths):
"f": "eighth",
}
group_to_path: Dict[str, List[str]] = {}
group_to_path: dict[str, list[str]] = {}
for path in unittest_paths:
norm_path = path.replace(":", "/").replace("\\", "/")

3
buildscripts/bazel_rules_mongo/codeowners/check_github_codeowner_errors.py
View File

@ -9,7 +9,6 @@ import requests
import structlog
import yaml
from typing import Dict
LOGGER = structlog.get_logger(__name__)
@ -195,7 +194,7 @@ def format_error(error: dict) -> str:
return "\n".join(parts)
def get_expansions(expansions_file: str) -> Dict[str, any]:
def get_expansions(expansions_file: str) -> dict[str, any]:
if not expansions_file:
return None

14
buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py
View File

@ -5,7 +5,7 @@ import subprocess
import sys
import tempfile
from functools import cache
from typing import Dict, List, Optional, Set, Tuple
from typing import Optional
import yaml
from codeowners.parsers import owners_v1, owners_v2
@ -21,12 +21,12 @@ parsers = {
class FileNode:
def __init__(self, directory: str):
self.dirs: Dict[str, FileNode] = {}
self.dirs: dict[str, FileNode] = {}
self.owners_file: Optional[str] = None
self.directory = directory
def add_file_to_tree(root_node: FileNode, file_parts: List[str]):
def add_file_to_tree(root_node: FileNode, file_parts: list[str]):
current_node = root_node
for i, dir in enumerate(file_parts[:-1]):
node_dirs = current_node.dirs
@ -42,7 +42,7 @@ def add_file_to_tree(root_node: FileNode, file_parts: List[str]):
current_node.owners_file = file_parts[-1]
def build_tree(files: List[str]) -> FileNode:
def build_tree(files: list[str]) -> FileNode:
root_node = FileNode("./")
for file in files:
file_parts = file.split("/")
@ -117,7 +117,7 @@ def validate_generated_codeowners(validator_path: str) -> int:
@cache
def get_unowned_and_default_owned_files(
codeowners_binary_path: str, codeowners_file: str = None
) -> Tuple[Set[str], Set[str]]:
) -> tuple[set[str], set[str]]:
temp_output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".txt")
temp_output_file.close()
default_owner = get_default_owner()
@ -327,7 +327,7 @@ def get_default_owner() -> Optional[str]:
@cache
def get_allowed_unowned_files() -> Set[str]:
def get_allowed_unowned_files() -> set[str]:
allowed_unowned_file_path = get_allowed_unowned_files_path()
if not allowed_unowned_file_path:
return set()
@ -367,7 +367,7 @@ def get_allowed_unowned_files() -> Set[str]:
return unowned_files
def add_allowed_unowned_files(output_lines: List[str]) -> None:
def add_allowed_unowned_files(output_lines: list[str]) -> None:
allowed_unowned_files = get_allowed_unowned_files()
if not allowed_unowned_files:
return

5
buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py
View File

@ -2,14 +2,13 @@ import glob
import os
import pathlib
from functools import cache
from typing import Dict, List
import yaml
# Parser for OWNERS.yml files version 1.0.0
class OwnersParserV1:
def parse(self, directory: str, owners_file_path: str, contents: Dict[str, any]) -> List[str]:
def parse(self, directory: str, owners_file_path: str, contents: dict[str, any]) -> list[str]:
lines = []
no_parent_owners = False
if "options" in contents:
@ -71,7 +70,7 @@ class OwnersParserV1:
return lines
@cache
def process_alias_import(self, path: str) -> Dict[str, List[str]]:
def process_alias_import(self, path: str) -> dict[str, list[str]]:
if not path.startswith("//"):
raise RuntimeError(
f"Alias file paths must start with // and be relative to the repo root: {path}"

1
buildscripts/bazel_rules_mongo/engflow_auth/engflow_auth.py
View File

@ -2,7 +2,6 @@
Install engflow_auth binary if not on system and authenticate if token is expired.
"""
import argparse
import hashlib
import json
import os

10
buildscripts/bazel_rules_mongo/utils/evergreen_git.py
View File

@ -1,13 +1,13 @@
import os
from functools import cache
from typing import Dict, List, Optional
from typing import Optional
import yaml
from git import Remote, Repo
@cache
def get_expansions(expansions_file: str) -> Dict[str, any]:
def get_expansions(expansions_file: str) -> dict[str, any]:
if not expansions_file:
return None
@ -89,7 +89,7 @@ def get_remote_branch_ref(repo: Repo, branch: str = None) -> str:
return repo.git.merge_base(local_head.hexsha, remote_head.hexsha)
def get_new_files(expansions_file: str = None, branch: str = None) -> List[str]:
def get_new_files(expansions_file: str = None, branch: str = None) -> list[str]:
# docs on the diff-filter are here https://www.kernel.org/pub/software/scm/git/docs/git-diff.html
# This gets added, renamed, and copied files from the git diff.
return get_changed_files(expansions_file, branch, diff_filter="ARC")
@ -126,7 +126,7 @@ def get_diff_revision(expansions_file: str = None, branch: str = None) -> str:
def get_changed_files(
expansions_file: str = None, branch: str = None, diff_filter: str | None = "d"
) -> List[str]:
) -> list[str]:
diff_commit = get_diff_revision(expansions_file, branch)
repo = Repo()
@ -151,7 +151,7 @@ def get_file_at_revision(file: str, revision: str) -> Optional[str]:
raise ex
def get_files_to_lint() -> List[str]:
def get_files_to_lint() -> list[str]:
# Returns all tracked files and unstaged files
repo = Repo()
# all tracked files by git

56
buildscripts/burn_in_tests.py
View File

@ -11,7 +11,7 @@ import subprocess
import sys
from abc import ABC, abstractmethod
from collections import defaultdict
from typing import Dict, List, NamedTuple, Optional, Set, Tuple
from typing import NamedTuple, Optional
import click
import structlog
@ -158,7 +158,7 @@ def is_file_a_test_file(file_path: str) -> bool:
return True
def find_excludes(selector_file: str) -> Tuple[List, List, List]:
def find_excludes(selector_file: str) -> tuple[list, list, list]:
"""Parse etc/burn_in_tests.yml. Returns lists of excluded suites, tasks & tests."""
if not selector_file:
@ -180,7 +180,7 @@ def find_excludes(selector_file: str) -> Tuple[List, List, List]:
)
def filter_tests(tests: Set[str], exclude_tests: List[str]) -> Set[str]:
def filter_tests(tests: set[str], exclude_tests: list[str]) -> set[str]:
"""
Exclude tests which have been denylisted.
@ -251,7 +251,7 @@ class SuiteToBurnInInfo(NamedTuple):
name: str
resmoke_args: str
tests: List[str]
tests: list[str]
class TaskToBurnInInfo(NamedTuple):
@ -263,13 +263,13 @@ class TaskToBurnInInfo(NamedTuple):
"""
display_task_name: str
suites: List[SuiteToBurnInInfo]
suites: list[SuiteToBurnInInfo]
@classmethod
def from_task(
cls,
task: VariantTask,
tests_by_suite: Dict[str, List[str]],
tests_by_suite: dict[str, list[str]],
) -> "TaskToBurnInInfo":
"""
Gather the information needed to run the given task.
@ -296,9 +296,9 @@ class TaskToBurnInInfo(NamedTuple):
def create_task_list(
evergreen_conf: EvergreenProjectConfig,
build_variant: str,
tests_by_suite: Dict[str, List[str]],
tests_by_suite: dict[str, list[str]],
exclude_tasks: [str],
) -> Dict[str, TaskToBurnInInfo]:
) -> dict[str, TaskToBurnInInfo]:
"""
Find associated tasks for the specified build_variant and suites.
@ -334,8 +334,8 @@ def create_task_list(
def _process_tests_by_suite(
task: VariantTask, tests_by_suite: Dict[str, List[str]]
) -> Dict[str, List[str]]:
task: VariantTask, tests_by_suite: dict[str, list[str]]
) -> dict[str, list[str]]:
"""Filter tests that should run under task according to build variant and task configuration."""
suite_to_run_options_map = task.combined_suite_to_resmoke_args_map
tests_by_suite_for_task = defaultdict(list)
@ -369,12 +369,12 @@ def _set_resmoke_cmd(repeat_config: RepeatConfig, resmoke_args: [str]) -> [str]:
def create_task_list_for_tests(
changed_tests: Set[str],
changed_tests: set[str],
build_variant: str,
evg_conf: EvergreenProjectConfig,
exclude_suites: Optional[List] = None,
exclude_tasks: Optional[List] = None,
) -> Dict[str, TaskToBurnInInfo]:
exclude_suites: Optional[list] = None,
exclude_tasks: Optional[list] = None,
) -> dict[str, TaskToBurnInInfo]:
"""
Create a list of tests by task for the given tests.
@ -402,8 +402,8 @@ def create_task_list_for_tests(
def create_tests_by_task(
build_variant: str,
evg_conf: EvergreenProjectConfig,
changed_tests: Set[str],
) -> Dict[str, TaskToBurnInInfo]:
changed_tests: set[str],
) -> dict[str, TaskToBurnInInfo]:
"""
Create a list of tests by task.
@ -429,7 +429,7 @@ def create_tests_by_task(
return {}
def run_tests(tests_by_task: Dict[str, TaskToBurnInInfo], resmoke_cmd: [str]) -> None:
def run_tests(tests_by_task: dict[str, TaskToBurnInInfo], resmoke_cmd: [str]) -> None:
"""
Run the given tests locally.
@ -488,7 +488,7 @@ class FileChangeDetector(ABC):
"""Interface to detect changes to files."""
@abstractmethod
def create_revision_map(self, repos: List[Repo]) -> RevisionMap:
def create_revision_map(self, repos: list[Repo]) -> RevisionMap:
"""
Create a map of the repos and the given revisions to diff against.
@ -497,7 +497,7 @@ class FileChangeDetector(ABC):
"""
raise NotImplementedError()
def find_changed_tests(self, repos: List[Repo]) -> Set[str]:
def find_changed_tests(self, repos: list[Repo]) -> set[str]:
"""
Find the changed tests.
@ -525,7 +525,7 @@ class LocalFileChangeDetector(FileChangeDetector):
"""
self.origin_rev = origin_rev
def create_revision_map(self, repos: List[Repo]) -> RevisionMap:
def create_revision_map(self, repos: list[Repo]) -> RevisionMap:
"""
Create a map of the repos and the given revisions to diff against.
@ -542,7 +542,7 @@ class BurnInExecutor(ABC):
"""An interface to execute discovered tests."""
@abstractmethod
def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None:
def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None:
"""
Execute the given tests in the given tasks.
@ -554,7 +554,7 @@ class BurnInExecutor(ABC):
class NopBurnInExecutor(BurnInExecutor):
"""A burn-in executor that displays results, but doesn't execute."""
def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None:
def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None:
"""
Execute the given tests in the given tasks.
@ -582,7 +582,7 @@ class LocalBurnInExecutor(BurnInExecutor):
self.resmoke_args = resmoke_args
self.repeat_config = repeat_config
def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None:
def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None:
"""
Execute the given tests in the given tasks.
@ -602,7 +602,7 @@ class DiscoveredSuite(BaseModel):
"""
suite_name: str
test_list: List[str]
test_list: list[str]
class DiscoveredTask(BaseModel):
@ -614,19 +614,19 @@ class DiscoveredTask(BaseModel):
"""
task_name: str
suites: List[DiscoveredSuite]
suites: list[DiscoveredSuite]
class DiscoveredTaskList(BaseModel):
"""Model for a list of discovered tasks."""
discovered_tasks: List[DiscoveredTask]
discovered_tasks: list[DiscoveredTask]
class YamlBurnInExecutor(BurnInExecutor):
"""A burn-in executor that outputs discovered tasks as YAML."""
def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None:
def execute(self, tests_by_task: dict[str, TaskToBurnInInfo]) -> None:
"""
Report the given tasks and their tests to stdout.
@ -667,7 +667,7 @@ class BurnInOrchestrator:
self.burn_in_executor = burn_in_executor
self.evg_conf = evg_conf
def burn_in(self, repos: List[Repo], build_variant: str) -> None:
def burn_in(self, repos: list[Repo], build_variant: str) -> None:
"""
Execute burn in tests for the given git repositories.

20
buildscripts/ciconfig/evergreen.py
View File

@ -12,7 +12,7 @@ import re
import shutil
import subprocess
import sys
from typing import Any, Dict, List, Optional, Set
from typing import Any, Optional
import structlog
@ -113,7 +113,7 @@ class EvergreenProjectConfig(object):
self.functions = self._conf["functions"]
@property
def task_names(self) -> List[str]:
def task_names(self) -> list[str]:
"""Get the list of task names."""
return list(self._tasks_by_name.keys())
@ -139,7 +139,7 @@ class EvergreenProjectConfig(object):
"""Return the variant with the given name as a Variant instance."""
return self._variants_by_name.get(variant_name)
def get_required_variants(self) -> Set[Variant]:
def get_required_variants(self) -> set[Variant]:
"""Get the list of required build variants."""
return {variant for variant in self.variants if variant.is_required_variant()}
@ -176,7 +176,7 @@ class Task(object):
return None
@property
def generate_resmoke_tasks_command(self) -> Optional[Dict[str, Any]]:
def generate_resmoke_tasks_command(self) -> Optional[dict[str, Any]]:
"""Return the 'generate resmoke tasks' command if found, or None."""
return self.find_func_command("generate resmoke tasks")
@ -186,7 +186,7 @@ class Task(object):
return self.generate_resmoke_tasks_command is not None
@property
def run_tests_command(self) -> Optional[Dict[str, Any]]:
def run_tests_command(self) -> Optional[dict[str, Any]]:
"""Return the 'run tests' command if found, or None."""
return self.find_func_command("run tests")
@ -196,7 +196,7 @@ class Task(object):
return self.run_tests_command is not None
@property
def initialize_multiversion_tasks_command(self) -> Optional[Dict[str, Any]]:
def initialize_multiversion_tasks_command(self) -> Optional[dict[str, Any]]:
"""Return the 'initialize multiversion tasks' command if found, or None."""
return self.find_func_command("initialize multiversion tasks")
@ -217,7 +217,7 @@ class Task(object):
return self.name[:-4]
def get_resmoke_command_vars(self) -> Dict[str, Any]:
def get_resmoke_command_vars(self) -> dict[str, Any]:
"""Get the vars for either 'generate resmoke tasks' or 'run tests', both eventually call resmoke.py."""
if self.is_run_tests_task:
return self.run_tests_command.get("vars", {})
@ -226,7 +226,7 @@ class Task(object):
return {}
def get_suite_names(self) -> List[str]:
def get_suite_names(self) -> list[str]:
"""Get the names of the resmoke.py suites from the task definition."""
command_vars = self.get_resmoke_command_vars()
@ -243,7 +243,7 @@ class Task(object):
raise ValueError(f"{self.name} task does not run a resmoke.py test suite")
@property
def suite_to_resmoke_args_map(self) -> Dict[str, str]:
def suite_to_resmoke_args_map(self) -> dict[str, str]:
"""Get the resmoke.py arguments from the task definition."""
output = {}
@ -439,7 +439,7 @@ class VariantTask(Task):
return f"{self.variant}: {self.name}"
@property
def combined_suite_to_resmoke_args_map(self) -> Dict[str, str]:
def combined_suite_to_resmoke_args_map(self) -> dict[str, str]:
"""Get the combined resmoke arguments.
This results from the concatenation of the task's resmoke_args parameter and the

16
buildscripts/clang_tidy.py
View File

@ -15,7 +15,7 @@ import sys
import time
from concurrent import futures
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Optional
import yaml
@ -41,12 +41,12 @@ for config in ["/tmp/compiledb-bin/.clang-tidy.strict", "bazel-bin/.clang-tidy.s
def _clang_tidy_executor(
clang_tidy_filename: Path,
clang_tidy_binary: str,
clang_tidy_cfg: Dict[str, Any],
clang_tidy_cfg: dict[str, Any],
output_dir: str,
show_stdout: bool,
mongo_check_module: str = "",
compile_commands: str = "compile_commands.json",
) -> Tuple[str, Optional[str]]:
) -> tuple[str, Optional[str]]:
clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent
os.makedirs(clang_tidy_parent_dir, exist_ok=True)
@ -92,7 +92,7 @@ def _clang_tidy_executor(
return proc.stdout.decode(locale.getpreferredencoding()), files_to_parse
def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int:
def _combine_errors(fixes_filename: str, files_to_parse: list[str]) -> int:
failed_files = 0
all_fixes = {}
@ -140,7 +140,7 @@ def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int:
return failed_files
def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str:
def __dedup_errors(clang_tidy_errors_threads: list[str]) -> str:
unique_single_errors = set()
for errs in clang_tidy_errors_threads:
if errs:
@ -211,7 +211,7 @@ def _run_tidy(args, parser_defaults):
)
sys.exit(1)
files_to_tidy: List[Path] = list()
files_to_tidy: list[Path] = list()
files_to_parse = list()
filtered_compile_commands = []
for file_doc in compile_commands:
@ -259,8 +259,8 @@ def _run_tidy(args, parser_defaults):
total_jobs = len(files_to_tidy)
workers = args.threads
clang_tidy_errors_futures: List[str] = []
clang_tidy_executor_futures: List[futures.ThreadPoolExecutor.submit] = []
clang_tidy_errors_futures: list[str] = []
clang_tidy_executor_futures: list[futures.ThreadPoolExecutor.submit] = []
# total completed tasks
tasks_completed = 0

4
buildscripts/client/jiraclient.py
View File

@ -2,7 +2,7 @@
import logging
from enum import Enum
from typing import Any, Dict, Iterable, Optional, Sequence
from typing import Any, Iterable, Optional, Sequence
from jira import JIRA, Issue
from jira.client import ResultList
@ -37,7 +37,7 @@ class JiraAuth(BaseSettings):
def get_token_auth(self) -> Optional[str]:
return self.pat
def get_oauth(self) -> Optional[Dict[str, Any]]:
def get_oauth(self) -> Optional[dict[str, Any]]:
if self.access_token and self.access_token_secret and self.consumer_key and self.key_cert:
return {
"access_token": self.access_token,

16
buildscripts/compare_evergreen_versions.py
View File

@ -37,7 +37,7 @@ from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional
from typing import Optional
# Add the repository root to sys.path to allow imports from buildscripts
_SCRIPT_DIR = Path(__file__).resolve().parent
@ -129,7 +129,7 @@ def _fetch_build_tasks(
build_variant_name: str,
build_id: str,
non_running_statuses: set[str],
) -> List[TaskStats]:
) -> list[TaskStats]:
"""
Fetch tasks for a single build variant.
@ -137,7 +137,7 @@ def _fetch_build_tasks(
"""
try:
build = evg_api.build_by_id(build_id)
tasks: List[Task] = build.get_tasks()
tasks: list[Task] = build.get_tasks()
task_stats_list = []
for task in tasks:
@ -172,7 +172,7 @@ def _fetch_build_tasks(
def get_version_tasks(
evg_api: RetryingEvergreenApi, version_id: str, max_workers: int = 10
) -> Dict[TaskKey, TaskStats]:
) -> dict[TaskKey, TaskStats]:
"""
Fetch all tasks for a given version using parallel requests.
@ -297,7 +297,7 @@ def compare_versions(
version_id_2: str,
task_filter: Optional[str] = None,
max_workers: int = 10,
) -> List[TaskComparison]:
) -> list[TaskComparison]:
"""
Compare tasks between two versions.
@ -376,7 +376,7 @@ def truncate_variant_name(variant_name: str) -> str:
def print_table_output(
comparisons: List[TaskComparison],
comparisons: list[TaskComparison],
version_id_1: str,
version_id_2: str,
min_diff_percent: float = 0.0,
@ -560,7 +560,7 @@ def print_table_output(
def print_json_output(
comparisons: List[TaskComparison],
comparisons: list[TaskComparison],
version_id_1: str,
version_id_2: str,
):
@ -601,7 +601,7 @@ def print_json_output(
def print_csv_output(
comparisons: List[TaskComparison],
comparisons: list[TaskComparison],
version_id_1: str,
version_id_2: str,
):

3
buildscripts/create_bazel_test_report.py
View File

@ -2,7 +2,6 @@ import json
import time
from glob import glob
from pathlib import Path
from typing import List
import typer
from typing_extensions import TypedDict
@ -24,7 +23,7 @@ class Result(TypedDict):
class Report(TypedDict):
"""Test report for Evergreen"""
results: List[Result]
results: list[Result]
def main(testlog_dir: str):

6
buildscripts/debugsymb_mapper.py
View File

@ -11,7 +11,7 @@ import subprocess
import sys
import time
from json import JSONDecoder
from typing import Dict, Generator, List, NamedTuple, Optional
from typing import Generator, NamedTuple, Optional
import requests
@ -33,7 +33,7 @@ class CmdClient:
"""Client to run commands."""
@staticmethod
def run(args: List[str]) -> str:
def run(args: list[str]) -> str:
"""
Run command with args.
@ -345,7 +345,7 @@ class Mapper:
tarball_full_path = download.download_from_s3(url)
return tarball_full_path
def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]:
def generate_build_id_mapping(self) -> Generator[dict[str, str], None, None]:
"""
Extract build id from binaries and creates new dict using them.

6
buildscripts/evergreen_gen_powercycle_tasks.py
View File

@ -2,7 +2,7 @@
"""Generate multiple powercycle tasks to run in evergreen."""
from collections import namedtuple
from typing import Any, List, Set, Tuple
from typing import Any
import click
from shrub.v2 import BuildVariant, ExistingTask, FunctionCall, ShrubProject, Task, TaskDependency
@ -68,14 +68,14 @@ def make_config(expansions_file: Any) -> Config:
)
def get_setup_commands() -> Tuple[List[FunctionCall], Set[TaskDependency]]:
def get_setup_commands() -> tuple[list[FunctionCall], set[TaskDependency]]:
"""Return setup commands."""
return [
FunctionCall("do setup"),
], {TaskDependency("archive_dist_test")}
def get_skip_compile_setup_commands() -> Tuple[List[FunctionCall], set]:
def get_skip_compile_setup_commands() -> tuple[list[FunctionCall], set]:
"""Return skip compile setup commands."""
return [
BuiltInCommand("manifest.load", {}),

4
buildscripts/evergreen_task_timeout.py
View File

@ -11,7 +11,7 @@ import shlex
import sys
from datetime import timedelta
from pathlib import Path
from typing import Dict, List, Optional
from typing import Optional
import inject
import structlog
@ -95,7 +95,7 @@ class TimeoutOverride(BaseModel):
class TimeoutOverrides(BaseModel):
"""Collection of timeout overrides to apply."""
overrides: Dict[str, List[TimeoutOverride]]
overrides: dict[str, list[TimeoutOverride]]
@classmethod
def from_yaml_file(cls, file_path: Path) -> "TimeoutOverrides":

6
buildscripts/fast_archive.py
View File

@ -8,7 +8,7 @@ import shutil
import subprocess
import sys
import time
from typing import Dict, List, Optional
from typing import Optional
import boto3
import requests
@ -28,7 +28,7 @@ def process_file(
file_number: int,
upload_name: str,
start_time: int,
) -> Optional[Dict[str, str]]:
) -> Optional[dict[str, str]]:
print(f"{file} started compressing at {time.time() - start_time}")
compressed_file = f"{file}.gz"
with open(file, "rb") as f_in:
@ -73,7 +73,7 @@ def process_file(
return task_artifact
def main(output_file: str, patterns: List[str], display_name: str, expansions_file: str) -> int:
def main(output_file: str, patterns: list[str], display_name: str, expansions_file: str) -> int:
if not output_file.endswith(".json"):
print("WARN: filename input should end with `.json`", file=sys.stderr)

16
buildscripts/fix_headers.py
View File

@ -12,7 +12,7 @@ import os
import pprint
import subprocess
import sys
from typing import Annotated, Dict, List, Optional, Tuple
from typing import Annotated, Optional
import typer
@ -72,7 +72,7 @@ def todo_comment(issue_key: str, header: str, new_dep: str, fg_label: str) -> No
bd_utils.bd_comment([fg_label], comment)
def useful_print(fixes: Dict) -> None:
def useful_print(fixes: dict) -> None:
for target, target_fixes in fixes.items():
print("-", target)
print(" Fixes:\n")
@ -92,7 +92,7 @@ class HeaderFixer:
self.team_issues = {}
def _query(
self, query: str, config: bool = False, args: List[str] = []
self, query: str, config: bool = False, args: list[str] = []
) -> subprocess.CompletedProcess:
query_cmd = "cquery"
@ -118,7 +118,7 @@ class HeaderFixer:
def _create_header_target(self):
pass
def _find_misplaced_headers(self, target: str) -> List[str]:
def _find_misplaced_headers(self, target: str) -> list[str]:
p = self._query(f"labels(hdrs,{target}{CC_LIB_SUFFIX})")
misplaced_headers = []
target_package = target.split(":")[0] + ":"
@ -134,7 +134,7 @@ class HeaderFixer:
return misplaced_headers
def _find_header_target(self, header: str) -> Tuple[Optional[str], bool]:
def _find_header_target(self, header: str) -> tuple[Optional[str], bool]:
potential_target = header.split(".")[0]
p = self._query(f"attr(srcs,{potential_target}.cpp,//...)")
target = None
@ -170,7 +170,7 @@ class HeaderFixer:
return False
def _fix_target(self, target: str) -> Dict:
def _fix_target(self, target: str) -> dict:
target_fixes = {"fixes": {}, "cycles": {}}
orphaned_headers = []
for hdr in self._find_misplaced_headers(target):
@ -208,7 +208,7 @@ class HeaderFixer:
print("\n".join(orphaned_headers))
return target_fixes
def _evaluate_target_expression(self, target_exp: str) -> List[str]:
def _evaluate_target_expression(self, target_exp: str) -> list[str]:
p = self._query(
f"filter('.*{CC_LIB_SUFFIX}$',kind(cc_library,deps({target_exp}, 1)))",
["--noimplicit_deps"],
@ -248,7 +248,7 @@ class HeaderFixer:
return ""
return issue.key
def fix_targets(self, target_exp: str) -> Dict:
def fix_targets(self, target_exp: str) -> dict:
fixes = {}
for target in self._evaluate_target_expression(target_exp):
fixes[target] = self._fix_target(target)

3
buildscripts/gather_failed_tests.py
View File

@ -3,7 +3,6 @@ import os
import shutil
import subprocess
from pathlib import Path
from typing import List
import typer
@ -27,7 +26,7 @@ def process_bep(bep_path):
return failed_tests, successful_tests
def _relink_binaries_with_symbols(failed_test_labels: List[str]):
def _relink_binaries_with_symbols(failed_test_labels: list[str]):
print("Rebuilding tests with --remote_download_outputs=toplevel...")
bazel_build_flags = ""
if os.path.isfile(".bazel_build_flags"):

4
buildscripts/generate_result_tasks.py
View File

@ -20,7 +20,7 @@ import os
import subprocess
import sys
from functools import cache
from typing import List, Optional
from typing import Optional
import runfiles
import typer
@ -131,7 +131,7 @@ def resolve_codeowners() -> dict[str, list[str]]:
return {}
def query_targets() -> List[str]:
def query_targets() -> list[str]:
try:
result = subprocess.run(
["bazel", "query", RESMOKE_TEST_QUERY],

8
buildscripts/get_agg_stages_no_authz.py
View File

@ -1,6 +1,6 @@
import os
import re
from typing import Optional, Set
from typing import Optional
MONGO_DOCUMENT_SOURCE_DIR = "src/mongo/db/pipeline"
@ -62,7 +62,7 @@ def check_file_pair(base_path: str) -> Optional[str]:
return None
def find_authz_opt_out_stages() -> Set[str]:
def find_authz_opt_out_stages() -> set[str]:
"""
Scans the pipeline directory to find aggregation stages that
opted out of authorization checks.
@ -74,8 +74,8 @@ def find_authz_opt_out_stages() -> Set[str]:
return set()
print(f"Scanning for aggregation stages in '{MONGO_DOCUMENT_SOURCE_DIR}'")
found_stages: Set[str] = set()
processed_bases: Set[str] = set()
found_stages: set[str] = set()
processed_bases: set[str] = set()
# Walk through directory
for root, _, files in os.walk(MONGO_DOCUMENT_SOURCE_DIR):

11
buildscripts/idl/check_stable_api_commands_have_idl_definitions.py
View File

@ -35,7 +35,6 @@ import logging
import os
import sys
from tempfile import TemporaryDirectory
from typing import Dict, List, Set
from pymongo import MongoClient
@ -65,8 +64,8 @@ def is_test_or_third_party_idl(idl_path: str) -> bool:
def get_command_definitions(
api_version: str, directory: str, import_directories: List[str]
) -> Dict[str, syntax.Command]:
api_version: str, directory: str, import_directories: list[str]
) -> dict[str, syntax.Command]:
"""Get parsed IDL definitions of commands in a given API version."""
LOGGER.info("Searching for command definitions in %s", directory)
@ -83,7 +82,7 @@ def get_command_definitions(
return idl_commands
def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> Set[str]:
def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> set[str]:
"""Get a list of commands in a given API version by calling listCommands."""
assert mongod_or_mongos in ("mongod", "mongos")
logging.info("Calling listCommands on %s", mongod_or_mongos)
@ -135,7 +134,7 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir:
fixture.teardown()
def assert_command_sets_equal(api_version: str, command_sets: Dict[str, Set[str]]):
def assert_command_sets_equal(api_version: str, command_sets: dict[str, set[str]]):
"""Check that all sources have the same set of commands for a given API version."""
LOGGER.info("Comparing %s command sets", len(command_sets))
for name, commands in command_sets.items():
@ -161,7 +160,7 @@ def assert_command_sets_equal(api_version: str, command_sets: Dict[str, Set[str]
)
def remove_skipped_commands(command_sets: Dict[str, Set[str]]):
def remove_skipped_commands(command_sets: dict[str, set[str]]):
"""Remove skipped commands from command_sets."""
skipped_commands = {
"testDeprecation",

7
buildscripts/idl/checkout_idl_files_from_past_releases.py
View File

@ -32,7 +32,6 @@ import os
import shutil
import sys
from subprocess import CalledProcessError, check_output
from typing import List
from packaging.version import Version
from retry import retry
@ -51,7 +50,7 @@ LOGGER_NAME = "checkout-idl"
LOGGER = logging.getLogger(LOGGER_NAME)
def get_tags() -> List[str]:
def get_tags() -> list[str]:
"""Get a list of git tags that the IDL compatibility script should check against."""
def gen_versions_and_tags():
@ -111,11 +110,11 @@ def _show_with_retry(tag: str, path: str):
@retry(tries=3, delay=5)
def _fetch_with_retry(tags: List[str]):
def _fetch_with_retry(tags: list[str]):
return check_output(["git", "fetch", "origin", *tags])
def make_idl_directories(tags: List[str], destination: str) -> None:
def make_idl_directories(tags: list[str], destination: str) -> None:
"""For each tag, construct a source tree containing only its IDL files."""
LOGGER.info("Clearing destination directory '%s'", destination)
shutil.rmtree(destination, ignore_errors=True)

9
buildscripts/idl/gen_all_feature_flag_list.py
View File

@ -32,7 +32,6 @@ Used by resmoke.py to run only feature flag tests.
import os
import sys
from typing import List
import typer
import yaml
@ -44,7 +43,7 @@ from buildscripts.idl import lib
from buildscripts.idl.idl import binder
def get_all_feature_flags_turned_on_by_default(idl_dirs: List[str] = None):
def get_all_feature_flags_turned_on_by_default(idl_dirs: list[str] = None):
"""Generate a list of all feature flags that default to true."""
all_flags = lib.get_all_feature_flags(idl_dirs)
@ -53,7 +52,7 @@ def get_all_feature_flags_turned_on_by_default(idl_dirs: List[str] = None):
]
def get_all_feature_flags_turned_off_by_default(idl_dirs: List[str] = None):
def get_all_feature_flags_turned_off_by_default(idl_dirs: list[str] = None):
"""Generate a list of all feature flags that default to false."""
all_flags = lib.get_all_feature_flags(idl_dirs)
all_default_false_flags = [
@ -70,7 +69,7 @@ def get_all_feature_flags_turned_off_by_default(idl_dirs: List[str] = None):
return list(set(all_default_false_flags) - set(force_disabled_flags))
def get_all_unreleased_ifr_feature_flags(idl_dirs: List[str] = None):
def get_all_unreleased_ifr_feature_flags(idl_dirs: list[str] = None):
"""Generate a list of all features flags in the 'in_development' incremental rollout phase."""
all_flags = lib.get_all_feature_flags(idl_dirs)
@ -81,7 +80,7 @@ def get_all_unreleased_ifr_feature_flags(idl_dirs: List[str] = None):
]
def write_feature_flags_to_file(flags: List[str], filename: str):
def write_feature_flags_to_file(flags: list[str], filename: str):
"""Helper function to write feature flags to a file."""
with open(filename, "w") as output_file:
output_file.write("\n".join(flags))

3
buildscripts/idl/gen_all_server_params_list.py
View File

@ -32,7 +32,6 @@ Used by DSI to conditionally allow configuration of internalQueryStatsRateLimit
import os
import sys
from typing import List
# Permit imports from "buildscripts".
sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../..")))
@ -41,7 +40,7 @@ from buildscripts.idl import lib
from buildscripts.idl.idl import parser
def gen_all_server_params(idl_dirs: List[str] = None):
def gen_all_server_params(idl_dirs: list[str] = None):
"""Generate a list of all server parameters."""
default_idl_dirs = ["src"]

14
buildscripts/idl/gen_dts.py
View File

@ -90,7 +90,7 @@ import shutil
import sys
from pathlib import Path
from types import FunctionType
from typing import Dict, Optional, Set, TypedDict
from typing import Optional, TypedDict
# Permit imports from "buildscripts".
sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../..")))
@ -100,9 +100,9 @@ from buildscripts.idl.idl.compiler import CompilerImportResolver
class Declarations(TypedDict):
structs: Set[str]
commands: Set[str]
enums: Set[str]
structs: set[str]
commands: set[str]
enums: set[str]
def object_to_dts(obj: common.SourceLocation):
@ -187,7 +187,7 @@ def object_to_dts(obj: common.SourceLocation):
def gen_dts_from_file(
input_file: str, existing_decls: Optional[Declarations] = None
) -> Dict[str, int]:
) -> dict[str, int]:
if not os.path.exists(input_file):
logging.error('File "%s" not found', args.input_file)
return {}
@ -204,7 +204,7 @@ def gen_dts(
exists_func: FunctionType = os.path.exists,
ignore_imported=False,
existing_decls: Optional[Declarations] = None,
) -> Dict[str, int]:
) -> dict[str, int]:
if parsed_doc.errors:
parsed_doc.errors.dump_errors()
@ -245,7 +245,7 @@ def gen_dts_files():
COLS = shutil.get_terminal_size().columns
MAX_FILENAME_LEN = COLS // 3
def print_progress(file, additions: Dict[str, int]) -> None:
def print_progress(file, additions: dict[str, int]) -> None:
file = str(file)
file = file if len(file) < MAX_FILENAME_LEN else f"{file[:MAX_FILENAME_LEN - 3]}..."
info = f"Parsed {file}: ".ljust(MAX_FILENAME_LEN + len("Parsed : ") + 1)

4
buildscripts/idl/idl/generator.py
View File

@ -36,7 +36,7 @@ import sys
import textwrap
from abc import ABCMeta, abstractmethod
from enum import Enum
from typing import List, cast
from typing import cast
from . import (
ast,
@ -552,7 +552,7 @@ class _CppFileWriterBase(object):
def get_all_structs(spec: ast.IDLBoundSpec):
return spec.structs + cast(List[ast.Struct], spec.commands)
return spec.structs + cast(list[ast.Struct], spec.commands)
def make_mod_tag(vis: str):

86
buildscripts/idl/idl_check_compatibility.py
View File

@ -42,7 +42,7 @@ import os
import sys
from dataclasses import dataclass
from enum import Enum
from typing import Dict, List, Optional, Set, Tuple, Union
from typing import Optional, Union
import yaml
from idl import common, errors, parser, syntax
@ -71,17 +71,17 @@ rules = load_rules_file()
# Load the subsections from the global "rules.yml" file into separate global variables.
# Any of the following assignments will fail if no rules exist for the provided key.
ALLOW_ANY_TYPE_LIST: List[str] = rules["ALLOW_ANY_TYPE_LIST"]
IGNORE_ANY_TO_NON_ANY_LIST: List[str] = rules["IGNORE_ANY_TO_NON_ANY_LIST"]
IGNORE_NON_ANY_TO_ANY_LIST: List[str] = rules["IGNORE_NON_ANY_TO_ANY_LIST"]
ALLOW_CPP_TYPE_CHANGE_LIST: List[str] = rules["ALLOW_CPP_TYPE_CHANGE_LIST"]
IGNORE_STABLE_TO_UNSTABLE_LIST: List[str] = rules["IGNORE_STABLE_TO_UNSTABLE_LIST"]
ALLOWED_STABLE_FIELDS_LIST: List[str] = rules["ALLOWED_STABLE_FIELDS_LIST"]
IGNORE_COMMANDS_LIST: List[str] = rules["IGNORE_COMMANDS_LIST"]
ALLOW_ANY_TYPE_LIST: list[str] = rules["ALLOW_ANY_TYPE_LIST"]
IGNORE_ANY_TO_NON_ANY_LIST: list[str] = rules["IGNORE_ANY_TO_NON_ANY_LIST"]
IGNORE_NON_ANY_TO_ANY_LIST: list[str] = rules["IGNORE_NON_ANY_TO_ANY_LIST"]
ALLOW_CPP_TYPE_CHANGE_LIST: list[str] = rules["ALLOW_CPP_TYPE_CHANGE_LIST"]
IGNORE_STABLE_TO_UNSTABLE_LIST: list[str] = rules["IGNORE_STABLE_TO_UNSTABLE_LIST"]
ALLOWED_STABLE_FIELDS_LIST: list[str] = rules["ALLOWED_STABLE_FIELDS_LIST"]
IGNORE_COMMANDS_LIST: list[str] = rules["IGNORE_COMMANDS_LIST"]
RENAMED_COMPLEX_ACCESS_CHECKS: dict[str, str] = rules["RENAMED_COMPLEX_ACCESS_CHECKS"]
ALLOWED_NEW_COMPLEX_ACCESS_CHECKS: dict[str, List[str]] = rules["ALLOWED_NEW_COMPLEX_ACCESS_CHECKS"]
CHANGED_ACCESS_CHECKS_TYPE: dict[str, List[str]] = rules["CHANGED_ACCESS_CHECKS_TYPE"]
ALLOW_FIELD_VALUE_REMOVAL_LIST: dict[str, List[str]] = rules["ALLOW_FIELD_VALUE_REMOVAL_LIST"]
ALLOWED_NEW_COMPLEX_ACCESS_CHECKS: dict[str, list[str]] = rules["ALLOWED_NEW_COMPLEX_ACCESS_CHECKS"]
CHANGED_ACCESS_CHECKS_TYPE: dict[str, list[str]] = rules["CHANGED_ACCESS_CHECKS_TYPE"]
ALLOW_FIELD_VALUE_REMOVAL_LIST: dict[str, list[str]] = rules["ALLOW_FIELD_VALUE_REMOVAL_LIST"]
SKIPPED_FILES = [
"unittest.idl",
@ -100,7 +100,7 @@ class AllowedNewPrivilege:
"""Represents a privilege check that should be ignored by the API compatibility checker."""
resource_pattern: str
action_type: List[str]
action_type: list[str]
agg_stage: Optional[str] = None
@classmethod
@ -108,7 +108,7 @@ class AllowedNewPrivilege:
return cls(privilege.resource_pattern, privilege.action_type, privilege.agg_stage)
ALLOWED_NEW_ACCESS_CHECK_PRIVILEGES: Dict[str, List[AllowedNewPrivilege]] = dict(
ALLOWED_NEW_ACCESS_CHECK_PRIVILEGES: dict[str, list[AllowedNewPrivilege]] = dict(
# Do not add any command other than the aggregate command or any privilege that is not required
# only by an aggregation stage not present in previously released versions.
aggregate=[],
@ -180,12 +180,12 @@ def is_stable(stability: Optional[str]) -> bool:
def get_new_commands(
ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: List[str]
) -> Tuple[Dict[str, syntax.Command], Dict[str, syntax.IDLParsedSpec], Dict[str, str]]:
ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: list[str]
) -> tuple[dict[str, syntax.Command], dict[str, syntax.IDLParsedSpec], dict[str, str]]:
"""Get new IDL commands and check validity."""
new_commands: Dict[str, syntax.Command] = dict()
new_command_file: Dict[str, syntax.IDLParsedSpec] = dict()
new_command_file_path: Dict[str, str] = dict()
new_commands: dict[str, syntax.Command] = dict()
new_command_file: dict[str, syntax.IDLParsedSpec] = dict()
new_command_file_path: dict[str, str] = dict()
for dirpath, _, filenames in os.walk(new_idl_dir):
for new_filename in filenames:
@ -265,8 +265,8 @@ def check_subset(
cmd_name: str,
field_name: str,
type_name: str,
sub_list: List[Union[str, syntax.EnumValue]],
super_list: List[Union[str, syntax.EnumValue]],
sub_list: list[Union[str, syntax.EnumValue]],
super_list: list[Union[str, syntax.EnumValue]],
file_path: str,
):
"""Check if sub_list is a subset of the super_list and log an error if not."""
@ -283,8 +283,8 @@ def check_superset(
ctxt: IDLCompatibilityContext,
cmd_name: str,
type_name: str,
super_list: List[Union[str, syntax.EnumValue]],
sub_list: List[Union[str, syntax.EnumValue]],
super_list: list[Union[str, syntax.EnumValue]],
sub_list: list[Union[str, syntax.EnumValue]],
file_path: str,
param_name: Optional[str],
is_command_parameter: bool,
@ -1165,7 +1165,7 @@ def check_param_or_type_validator(
# SERVER-71601.
#
# Do not add additional parameters to this list.
ignore_validator_check_list: List[str] = []
ignore_validator_check_list: list[str] = []
if new_field.validator:
if old_field.validator:
@ -1493,8 +1493,8 @@ def check_namespace(
def check_error_reply(
old_basic_types_path: str,
new_basic_types_path: str,
old_import_directories: List[str],
new_import_directories: List[str],
old_import_directories: list[str],
new_import_directories: list[str],
) -> IDLCompatibilityErrorCollection:
"""Check IDL compatibility between old and new ErrorReply."""
old_idl_dir = os.path.dirname(old_basic_types_path)
@ -1546,8 +1546,8 @@ def check_error_reply(
def split_complex_checks(
complex_checks: List[syntax.AccessCheck],
) -> Tuple[List[str], List[syntax.Privilege]]:
complex_checks: list[syntax.AccessCheck],
) -> tuple[list[str], list[syntax.Privilege]]:
"""Split a list of AccessCheck into checks and privileges."""
checks = [x.check for x in complex_checks if x.check is not None]
privileges = [x.privilege for x in complex_checks if x.privilege is not None]
@ -1566,8 +1566,8 @@ def map_complex_access_check_name(name: str) -> str:
def check_complex_checks(
ctxt: IDLCompatibilityContext,
old_complex_checks: List[syntax.AccessCheck],
new_complex_checks: List[syntax.AccessCheck],
old_complex_checks: list[syntax.AccessCheck],
new_complex_checks: list[syntax.AccessCheck],
cmd: syntax.Command,
new_idl_file_path: str,
) -> None:
@ -1615,10 +1615,10 @@ def check_complex_checks(
def split_complex_checks_agg_stages(
complex_checks: List[syntax.AccessCheck],
) -> Dict[str, List[syntax.AccessCheck]]:
complex_checks: list[syntax.AccessCheck],
) -> dict[str, list[syntax.AccessCheck]]:
"""Split a list of AccessChecks into a map keyed by aggregation stage (defaults to None)."""
complex_checks_agg_stages: Dict[str, List[syntax.AccessCheck]] = dict()
complex_checks_agg_stages: dict[str, list[syntax.AccessCheck]] = dict()
for access_check in complex_checks:
agg_stage = None
if access_check.privilege is not None:
@ -1632,8 +1632,8 @@ def split_complex_checks_agg_stages(
def check_complex_checks_agg_stages(
ctxt: IDLCompatibilityContext,
old_complex_checks: List[syntax.AccessCheck],
new_complex_checks: List[syntax.AccessCheck],
old_complex_checks: list[syntax.AccessCheck],
new_complex_checks: list[syntax.AccessCheck],
cmd: syntax.Command,
new_idl_file_path: str,
) -> None:
@ -1711,8 +1711,8 @@ def check_security_access_checks(
def check_compatibility(
old_idl_dir: str,
new_idl_dir: str,
old_import_directories: List[str],
new_import_directories: List[str],
old_import_directories: list[str],
new_import_directories: list[str],
) -> IDLCompatibilityErrorCollection:
"""Check IDL compatibility between old and new IDL commands."""
ctxt = IDLCompatibilityContext(old_idl_dir, new_idl_dir, IDLCompatibilityErrorCollection())
@ -1724,7 +1724,7 @@ def check_compatibility(
# Check new commands' compatibility with old ones.
# Note, a command can be added to V1 at any time, it's ok if a
# new command has no corresponding old command.
old_commands: Dict[str, syntax.Command] = dict()
old_commands: dict[str, syntax.Command] = dict()
for dirpath, _, filenames in os.walk(old_idl_dir):
for old_filename in filenames:
if not old_filename.endswith(".idl") or old_filename in SKIPPED_FILES:
@ -1827,11 +1827,11 @@ def check_compatibility(
def get_generic_arguments(
gen_args_file_path: str, includes: List[str]
) -> Tuple[Set[str], Set[str]]:
gen_args_file_path: str, includes: list[str]
) -> tuple[set[str], set[str]]:
"""Get arguments and reply fields from generic_argument.idl and check validity."""
arguments: Set[str] = set()
reply_fields: Set[str] = set()
arguments: set[str] = set()
reply_fields: set[str] = set()
with open(gen_args_file_path) as gen_args_file:
parsed_idl_file = parser.parse(
@ -1875,8 +1875,8 @@ def get_generic_arguments(
def check_generic_arguments_compatibility(
old_gen_args_file_path: str,
new_gen_args_file_path: str,
old_includes: List[str],
new_includes: List[str],
old_includes: list[str],
new_includes: list[str],
) -> IDLCompatibilityErrorCollection:
"""Check IDL compatibility between old and new generic_argument.idl files."""
# IDLCompatibilityContext takes in both 'old_idl_dir' and 'new_idl_dir',

8
buildscripts/idl/idl_compatibility_errors.py
View File

@ -34,7 +34,7 @@ Common error handling code for IDL compatibility checker.
import inspect
import sys
from typing import List, Optional
from typing import Optional
# Public error codes used by IDL compatibility checker.
# Used by tests cases to validate expected errors are thrown in negative tests.
@ -188,7 +188,7 @@ class IDLCompatibilityErrorCollection(object):
def __init__(self) -> None:
"""Initialize IDLCompatibilityErrorCollection."""
self._errors: List[IDLCompatibilityError] = []
self._errors: list[IDLCompatibilityError] = []
def add(
self,
@ -236,11 +236,11 @@ class IDLCompatibilityErrorCollection(object):
assert error is not None
return error
def get_all_errors_by_command_name(self, command_name: str) -> List[IDLCompatibilityError]:
def get_all_errors_by_command_name(self, command_name: str) -> list[IDLCompatibilityError]:
"""Get all the errors in the error collection with the command command_name."""
return [a for a in self._errors if a.command_name == command_name]
def to_list(self) -> List[str]:
def to_list(self) -> list[str]:
"""Return a list of formatted error messages."""
return [str(error) for error in self._errors]

7
buildscripts/idl/lib.py
View File

@ -27,13 +27,12 @@
"""Library functions and utility methods used across user-facing IDL scripts."""
import os
from typing import List, Set
from buildscripts.idl.idl import parser, syntax
from buildscripts.idl.idl.compiler import CompilerImportResolver
def list_idls(directory: str) -> Set[str]:
def list_idls(directory: str) -> set[str]:
"""Find all IDL files in the current directory."""
return {
os.path.join(dirpath, filename)
@ -43,7 +42,7 @@ def list_idls(directory: str) -> Set[str]:
}
def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedSpec:
def parse_idl(idl_path: str, import_directories: list[str]) -> syntax.IDLParsedSpec:
"""Parse an IDL file or throw an error."""
parsed_doc = parser.parse(open(idl_path), idl_path, CompilerImportResolver(import_directories))
@ -65,7 +64,7 @@ def is_third_party_idl(idl_path: str) -> bool:
return False
def get_all_feature_flags(idl_dirs: List[str] = None):
def get_all_feature_flags(idl_dirs: list[str] = None):
"""Generate a dict of all feature flags with their default value."""
default_idl_dirs = ["src", "buildscripts"]

12
buildscripts/idl/tests/test_gendts.py
View File

@ -37,7 +37,7 @@ import unittest
from contextlib import contextmanager
from io import StringIO
from types import SimpleNamespace
from typing import DefaultDict, Iterable, Tuple
from typing import Iterable
# import package so that it works regardless of whether we run as a module or file
if __package__ is None:
@ -48,6 +48,8 @@ else:
# Permit imports from "buildscripts".
sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), "../../../../")))
from collections import defaultdict
from buildscripts.idl.gen_dts import gen_dts, object_to_dts, parser, syntax
from buildscripts.idl.idl.compiler import CompilerImportResolver
@ -78,7 +80,7 @@ class TestGenDTS(testcase.IDLTestcase):
class MockFilesystem:
def __init__(self) -> None:
self.files = DefaultDict(lambda: "")
self.files = defaultdict(lambda: "")
@contextmanager
def open(self, path: str, mode: str):
@ -120,13 +122,13 @@ class TestGenDTS(testcase.IDLTestcase):
self.assertEqual(mock_fs.files["src/mongo/shell/enums_gen.d.ts"], "\n".join(enums))
self.assertEqual(mock_fs.files["src/mongo/shell/commands_gen.d.ts"], "\n".join(commands))
def _get_generic_idl_with_header(self) -> Tuple[str]:
def _get_generic_idl_with_header(self) -> tuple[str]:
"""Returns a header necessary to use commands in an IDL."""
return ("global:", " cpp_namespace: 'mongo'", "", "imports: []", "")
def _get_command_idl_text(
self, name: str, description: str, namespace: str = "ignored"
) -> Tuple[str]:
) -> tuple[str]:
"""Returns a command IDL declaration with a given name, description, and namespace."""
return (
"commands:",
@ -140,7 +142,7 @@ class TestGenDTS(testcase.IDLTestcase):
def _get_command_idl_with_header(
self, name: str, description: str, namespace: str = "ignored"
) -> Tuple[str]:
) -> tuple[str]:
"""Returns a command IDL declaration with a given name, description, and namespace, with the necessary headers."""
return (
"global:",

24
buildscripts/jepsen_report.py
View File

@ -5,7 +5,7 @@ import os
import re
import sys
from datetime import datetime, timezone
from typing import List, Optional, Tuple
from typing import Optional
import click
from typing_extensions import TypedDict
@ -16,10 +16,10 @@ from buildscripts.simple_report import Report, Result
class ParserOutput(TypedDict):
"""Result of parsing jepsen log file. Each List[str] is a list of test names."""
success: List[str]
unknown: List[str]
crashed: List[str]
failed: List[str]
success: list[str]
unknown: list[str]
crashed: list[str]
failed: list[str]
start: int
end: int
elapsed: int
@ -41,7 +41,7 @@ def _time_parse(time: str):
return date.replace(microsecond=microseconds, tzinfo=timezone.utc)
def _calc_time_from_log(log: str) -> Tuple[int, int, int]:
def _calc_time_from_log(log: str) -> tuple[int, int, int]:
if not log:
return (0, 0, 0)
start_time = None
@ -67,7 +67,7 @@ UNKNOWN_RE = re.compile("([0-9]+) unknown")
FAIL_RE = re.compile("([0-9]+) failures")
def parse(text: List[str]) -> ParserOutput: # noqa: D406
def parse(text: list[str]) -> ParserOutput: # noqa: D406
"""Given a List of strings representing jepsen log file split by newlines, return the ParserOutput struct.
Args:
@ -80,10 +80,10 @@ def parse(text: List[str]) -> ParserOutput: # noqa: D406
AssertionError: If there is a mismatch between the count of matches and the corresponding test list.
"""
successful_tests: List[str] = []
indeterminate_tests: List[str] = []
crashed_tests: List[str] = []
failed_tests: List[str] = []
successful_tests: list[str] = []
indeterminate_tests: list[str] = []
crashed_tests: list[str] = []
failed_tests: list[str] = []
target = None
success_table_matches = 0
unknown_table_matches = 0
@ -258,7 +258,7 @@ def report(
)
def _get_log_lines(filename: str) -> List[str]:
def _get_log_lines(filename: str) -> list[str]:
with open(filename) as fh:
return fh.read().splitlines()

10
buildscripts/large_file_check.py
View File

@ -8,7 +8,7 @@ import os
import pathlib
import sys
import textwrap
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Optional
import structlog
from git import Repo
@ -27,7 +27,7 @@ from buildscripts.patch_builds.change_data import (
# Console renderer for structured logging
def renderer(_logger: logging.Logger, _name: str, eventdict: Dict[Any, Any]) -> str:
def renderer(_logger: logging.Logger, _name: str, eventdict: dict[Any, Any]) -> str:
if "files" in eventdict:
return "{event}: {files}".format(**eventdict)
if "repo" in eventdict:
@ -54,7 +54,7 @@ LOGGER = structlog.get_logger(__name__)
MONGO_REVISION_ENV_VAR = "REVISION"
def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]:
def _get_repos_and_revisions() -> tuple[list[Repo], RevisionMap]:
"""Get the repo object and a map of revisions to compare against."""
repo_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", None)
@ -67,7 +67,7 @@ def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]:
return repos, revision_map
def git_changed_files(excludes: List[pathlib.Path]) -> List[pathlib.Path]:
def git_changed_files(excludes: list[pathlib.Path]) -> list[pathlib.Path]:
"""
Get the files that have changes since the last git commit.
@ -95,7 +95,7 @@ def git_changed_files(excludes: List[pathlib.Path]) -> List[pathlib.Path]:
return files
def diff_file_sizes(size_limit: int, excludes: Optional[List[str]] = None) -> List[pathlib.Path]:
def diff_file_sizes(size_limit: int, excludes: Optional[list[str]] = None) -> list[pathlib.Path]:
if excludes is None:
excludes = []

3
buildscripts/legacy_commands_check.py
View File

@ -4,7 +4,6 @@
import os
import re
import sys
from typing import List, Tuple
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
@ -21,7 +20,7 @@ LEGACY_TYPES = [
]
def check_file_for_legacy_type(modified_lines: List[Tuple[int, str]]) -> bool:
def check_file_for_legacy_type(modified_lines: list[tuple[int, str]]) -> bool:
"""Return false if a file defines a legacy command."""
file_has_legacy_type = False

24
buildscripts/lint_markdown_links.py
View File

@ -120,7 +120,7 @@ import re
import sys
import urllib.parse
from dataclasses import dataclass
from typing import Iterable, List, Optional, Tuple
from typing import Iterable, Optional
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$")
HTML_ANCHOR_RE = re.compile(r'<a\s+(?:name|id)=["\']([^"\']+)["\']\s*>\s*</a>?', re.IGNORECASE)
@ -273,8 +273,8 @@ def is_http_url(url: str) -> bool:
return url.startswith("http://") or url.startswith("https://")
def find_markdown_files(root: str) -> List[str]:
files: List[str] = []
def find_markdown_files(root: str) -> list[str]:
files: list[str] = []
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
if fn.lower().endswith(".md"):
@ -282,8 +282,8 @@ def find_markdown_files(root: str) -> List[str]:
return files
def parse_links(file_path: str) -> List[Tuple[int, str, str]]:
links: List[Tuple[int, str, str]] = []
def parse_links(file_path: str) -> list[tuple[int, str, str]]:
links: list[tuple[int, str, str]] = []
try:
with open(file_path, "r", encoding="utf-8") as f:
in_fence = False
@ -546,11 +546,11 @@ def validate_link(current_file: str, line: int, text: str, target: str) -> Optio
return None
def lint_files(files: Iterable[str], workers: int) -> List[LinkIssue]:
issues: List[LinkIssue] = []
def lint_files(files: Iterable[str], workers: int) -> list[LinkIssue]:
issues: list[LinkIssue] = []
def process(file_path: str) -> List[LinkIssue]:
file_issues: List[LinkIssue] = []
def process(file_path: str) -> list[LinkIssue]:
file_issues: list[LinkIssue] = []
links = parse_links(file_path)
for line, text, target in links:
issue = validate_link(file_path, line, text, target)
@ -566,7 +566,7 @@ def lint_files(files: Iterable[str], workers: int) -> List[LinkIssue]:
return issues
def main(argv: List[str]) -> int:
def main(argv: list[str]) -> int:
ap = argparse.ArgumentParser(description="Markdown link linter for src/mongo markdown files.")
ap.add_argument("--root", default="src/mongo", help="Root directory to scan")
ap.add_argument(
@ -646,7 +646,7 @@ def main(argv: List[str]) -> int:
fix_count = 0
# Group issues by file for editing
issues_by_file: dict[str, List[LinkIssue]] = {}
issues_by_file: dict[str, list[LinkIssue]] = {}
for iss in issues:
issues_by_file.setdefault(iss.file, []).append(iss)
@ -677,7 +677,7 @@ def main(argv: List[str]) -> int:
# Deduplicate identical (message, target) to avoid repeated work (retain first occurrence)
seen_sig = set()
deduped: List[LinkIssue] = []
deduped: list[LinkIssue] = []
for iss in file_issues:
sig = (iss.message, iss.target)
if sig in seen_sig:

8
buildscripts/linter/filediff.py
View File

@ -2,7 +2,7 @@
import os
import sys
from typing import Callable, Dict, List, Tuple
from typing import Callable
import structlog
from git import Repo
@ -23,7 +23,7 @@ LOGGER = structlog.get_logger(__name__)
MONGO_REVISION_ENV_VAR = "REVISION"
def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]:
def _get_repos_and_revisions() -> tuple[list[Repo], RevisionMap]:
"""Get the repo object and a map of revisions to compare against."""
repos = [Repo(git.get_base_dir())]
@ -42,7 +42,7 @@ def _filter_file(filename: str, is_interesting_file: Callable[[str], bool]) -> b
return os.path.exists(filename) and is_interesting_file(filename)
def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> List[str]:
def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) -> list[str]:
"""
Get the files that have changes since the last git commit.
@ -63,7 +63,7 @@ def gather_changed_files_for_lint(is_interesting_file: Callable[[str], bool]) ->
def gather_changed_files_with_lines(
is_interesting_file: Callable[[str], bool],
) -> Dict[str, List[Tuple[int, str]]]:
) -> dict[str, list[tuple[int, str]]]:
"""
Get the files that have changes since the last git commit, along with details of the specific lines that have changed.

4
buildscripts/linter/git.py
View File

@ -6,7 +6,7 @@ import itertools
import os
import re
from pathlib import Path
from typing import Callable, List
from typing import Callable
from buildscripts.linter import git_base as _git
@ -29,7 +29,7 @@ def get_base_dir():
return os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
def get_repos() -> List[Repo]:
def get_repos() -> list[Repo]:
"""Get a list of Repos to check linters for."""
return [Repo(get_base_dir())]

6
buildscripts/linter/pyrightlinter.py
View File

@ -1,7 +1,5 @@
"""Pyright linter support module."""
from typing import List
from . import base
@ -12,10 +10,10 @@ class PyrightLinter(base.LinterBase):
"""Create a Pyright linter."""
super(PyrightLinter, self).__init__("pyright", "1.1.393")
def get_lint_version_cmd_args(self) -> List[str]:
def get_lint_version_cmd_args(self) -> list[str]:
"""Get the command to run a version check."""
return ["--version"]
def get_lint_cmd_args(self, files: List[str]) -> List[str]:
def get_lint_cmd_args(self, files: list[str]) -> list[str]:
"""Get the command to run a check."""
return files if files else ["."]

22
buildscripts/mongosymb.py
View File

@ -29,7 +29,7 @@ from abc import ABC, abstractmethod
from collections import OrderedDict
from datetime import timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
import requests
from tenacity import Retrying, retry_if_result, stop_after_delay, wait_fixed
@ -52,7 +52,7 @@ class DbgFileResolver(ABC):
"""Base gdb path resolver class."""
@abstractmethod
def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]:
def get_dbg_file(self, soinfo: dict[str, Any]) -> Union[str, None]:
"""
To get path for given build info.
@ -124,7 +124,7 @@ class CachedResults(object):
Idea is to allow only N items to be present in cache at a time and eliminate extra items on the go.
"""
def __init__(self, max_cache_size: int, initial_cache: Dict[str, str] = None):
def __init__(self, max_cache_size: int, initial_cache: dict[str, str] = None):
"""
Initialize instance.
@ -134,7 +134,7 @@ class CachedResults(object):
self._max_cache_size = max_cache_size
self._cached_results = OrderedDict(initial_cache or {})
def insert(self, key: str, value: str) -> Dict[str, str] or None:
def insert(self, key: str, value: str) -> dict[str, str] or None:
"""
Insert new data into cache.
@ -302,7 +302,7 @@ class PathResolver(DbgFileResolver):
"""
return self._cached_results.get(key)
def add_to_cache(self, key: str, value: str) -> Dict[str, str]:
def add_to_cache(self, key: str, value: str) -> dict[str, str]:
"""
Add new value to cache.
@ -484,7 +484,7 @@ def parse_input(trace_doc, dbg_path_resolver):
return frames
def get_version(trace_doc: Dict[str, Any]) -> Optional[str]:
def get_version(trace_doc: dict[str, Any]) -> Optional[str]:
"""
Get version from trace doc.
@ -569,8 +569,8 @@ def symbolize_frames(
def preprocess_frames(
dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], input_format: str
) -> List[Dict[str, Any]]:
dbg_path_resolver: DbgFileResolver, trace_doc: dict[str, Any], input_format: str
) -> list[dict[str, Any]]:
"""
Process the paths in frame objects.
@ -592,7 +592,7 @@ def preprocess_frames(
return frames
def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool:
def has_high_not_found_paths_ratio(frames: list[dict[str, Any]]) -> bool:
"""
Check whether not found paths in frames ratio is higher than 0.5.
@ -606,10 +606,10 @@ def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool:
def preprocess_frames_with_retries(
dbg_path_resolver: DbgFileResolver,
trace_doc: Dict[str, Any],
trace_doc: dict[str, Any],
input_format: str,
total_seconds_for_retries: int = 0,
) -> List[Dict[str, Any]]:
) -> list[dict[str, Any]]:
"""
Process the paths in frame objects.

13
buildscripts/monitor_build_status/cli.py
View File

@ -4,7 +4,6 @@ import os
import sys
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Dict, List, Tuple
import structlog
import typer
@ -44,7 +43,7 @@ class CodeMergeStatus(Enum):
GREEN = "GREEN"
@classmethod
def from_threshold_percentages(cls, threshold_percentages: List[float]) -> CodeMergeStatus:
def from_threshold_percentages(cls, threshold_percentages: list[float]) -> CodeMergeStatus:
if any(percentage > 100 for percentage in threshold_percentages):
return cls.RED
return cls.GREEN
@ -75,7 +74,7 @@ class MonitorBuildStatusOrchestrator:
summaries = ""
for scopes_config in notification_config.scopes:
scope_percentages: Dict[str, List[float]] = {}
scope_percentages: dict[str, list[float]] = {}
issue_report = self._make_report(scopes_config)
issue_count_status_msg, issue_count_percentages = self._get_issue_counts_status(
@ -117,9 +116,9 @@ class MonitorBuildStatusOrchestrator:
def _get_issue_counts_status(
self, scope_name: str, issue_report: IssueReport, notification_config: NotificationsConfig
) -> Tuple[str, Dict[str, List[float]]]:
) -> tuple[str, dict[str, list[float]]]:
now = datetime.now(timezone.utc)
percentages: Dict[str, List[float]] = {}
percentages: dict[str, list[float]] = {}
headers = [scope_name, "Hot Issues", "Cold Issues"]
table_data = []
@ -129,7 +128,7 @@ class MonitorBuildStatusOrchestrator:
hot_issue_count: int,
cold_issue_count: int,
thresholds: IssueThresholds,
slack_tags: List[str],
slack_tags: list[str],
) -> None:
if all(count == 0 for count in [hot_issue_count, cold_issue_count]):
return
@ -242,7 +241,7 @@ class MonitorBuildStatusOrchestrator:
return message, percentages
@staticmethod
def _summarize(scope_name: str, scope_percentages: Dict[str, List[float]]) -> str:
def _summarize(scope_name: str, scope_percentages: dict[str, list[float]]) -> str:
summary = f"`SUMMARY [{scope_name}]`"
red_sub_scopes = []

26
buildscripts/monitor_build_status/code_lockdown_config.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
from copy import deepcopy
from typing import List, Optional
from typing import Optional
import yaml
from pydantic import BaseModel
@ -40,33 +40,33 @@ class ScopesConfig(BaseModel):
class SlackConfig(BaseModel):
channel: str
overall_scope_tags: List[str]
overall_scope_tags: list[str]
message_footer: str
short_issue_data_table: bool = False
class NotificationsConfig(BaseModel):
scopes: List[ScopesConfig]
scopes: list[ScopesConfig]
thresholds: ThresholdsConfig
slack: SlackConfig
class TeamConfig(BaseModel):
name: str
slack_tags: Optional[List[str]]
slack_tags: Optional[list[str]]
thresholds: Optional[ThresholdOverride]
class GroupConfig(BaseModel):
name: str
teams: List[str]
slack_tags: Optional[List[str]]
teams: list[str]
slack_tags: Optional[list[str]]
class CodeLockdownConfig(BaseModel):
notifications: List[NotificationsConfig]
teams: List[TeamConfig]
groups: List[GroupConfig]
notifications: list[NotificationsConfig]
teams: list[TeamConfig]
groups: list[GroupConfig]
@classmethod
def from_yaml_config(cls, file_path: str) -> CodeLockdownConfig:
@ -79,11 +79,11 @@ class CodeLockdownConfig(BaseModel):
with open(file_path, encoding="utf8") as file_handler:
return cls(**yaml.safe_load(file_handler))
def get_all_group_names(self) -> List[str]:
def get_all_group_names(self) -> list[str]:
"""Get all group names."""
return [group.name for group in self.groups]
def get_group_teams(self, group_name: str) -> List[str]:
def get_group_teams(self, group_name: str) -> list[str]:
"""
Get group teams.
@ -96,7 +96,7 @@ class CodeLockdownConfig(BaseModel):
return []
def get_group_slack_tags(self, group_name: str) -> List[str]:
def get_group_slack_tags(self, group_name: str) -> list[str]:
"""
Get group slack tags.
@ -109,7 +109,7 @@ class CodeLockdownConfig(BaseModel):
return []
def get_team_slack_tags(self, team_name: str) -> List[str]:
def get_team_slack_tags(self, team_name: str) -> list[str]:
"""
Get team slack tags.

10
buildscripts/monitor_build_status/issue_report.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from datetime import datetime, timezone
from enum import Enum
from typing import Dict, Iterable, List, NamedTuple, Optional, Set
from typing import Iterable, NamedTuple, Optional
from buildscripts.monitor_build_status.jira_service import IssueTuple
@ -13,8 +13,8 @@ class IssueCategory(str, Enum):
class CategorizedIssues(NamedTuple):
hot: Set[IssueTuple]
cold: Set[IssueTuple]
hot: set[IssueTuple]
cold: set[IssueTuple]
@classmethod
def empty(cls) -> CategorizedIssues:
@ -49,7 +49,7 @@ class CategorizedIssues(NamedTuple):
class IssueReport(NamedTuple):
team_reports: Dict[str, CategorizedIssues]
team_reports: dict[str, CategorizedIssues]
@classmethod
def empty(cls) -> IssueReport:
@ -76,7 +76,7 @@ class IssueReport(NamedTuple):
self,
category: IssueCategory,
include_items_older_than_time: Optional[datetime] = None,
assigned_teams: Optional[List[str]] = None,
assigned_teams: Optional[list[str]] = None,
) -> int:
"""
Calculate Issue count for a given criteria.

4
buildscripts/monitor_build_status/jira_service.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from datetime import datetime
from enum import Enum
from typing import Any, List, NamedTuple, Optional
from typing import Any, NamedTuple, Optional
import dateutil.parser
from jira import Issue
@ -63,7 +63,7 @@ class JiraService:
def __init__(self, jira_client: JiraClient) -> None:
self.jira_client = jira_client
def fetch_issues(self, query: str) -> List[IssueTuple]:
def fetch_issues(self, query: str) -> list[IssueTuple]:
"""
Fetch issues from Jira and transform it into consumable form.

4
buildscripts/monitor_mongo_fork_10gen.py
View File

@ -1,6 +1,6 @@
import argparse
import sys
from typing import List, Optional
from typing import Optional
from github import Github, GithubException, GithubIntegration
from simple_report import make_report, put_report
@ -50,7 +50,7 @@ def get_users_who_forked_mongo_repo(owner: str, repo: str, token: str) -> list[s
return [fork.owner.login for fork in repository.get_forks() if not fork.archived]
def are_users_members_of_org(users: List[str], org: str, token: str) -> List[str]: # noqa: D406
def are_users_members_of_org(users: list[str], org: str, token: str) -> list[str]: # noqa: D406
"""
Check if users are members of a particular organization.

34
buildscripts/package_test.py
View File

@ -13,7 +13,7 @@ import traceback
import uuid
from concurrent import futures
from pathlib import Path
from typing import Any, Dict, Generator, List, Optional, Set, Tuple
from typing import Any, Generator, Optional
from urllib.parse import urlparse
import docker
@ -313,10 +313,10 @@ OS_DOCKER_LOOKUP = {
}
# These versions are marked "current" but in fact are EOL
VERSIONS_TO_SKIP: Set[str] = set(
VERSIONS_TO_SKIP: set[str] = set(
["3.0.15", "3.2.22", "3.4.24", "3.6.23", "4.0.28", "4.2.24", "4.2.25", "4.4.29", "6.3.2"]
)
DISABLED_TESTS: Set[Tuple[str, str]] = set()
DISABLED_TESTS: set[tuple[str, str]] = set()
VALID_TAR_DIRECTORY_ARCHITECTURES = [
"linux-aarch64",
@ -340,10 +340,10 @@ class Test:
package_manager: str = dataclasses.field(default="", repr=False)
update_command: str = dataclasses.field(default="", repr=False)
install_command: str = dataclasses.field(default="", repr=False)
base_packages: List[str] = dataclasses.field(default_factory=list)
base_packages: list[str] = dataclasses.field(default_factory=list)
python_command: str = dataclasses.field(default="", repr=False)
packages_urls: List[str] = dataclasses.field(default_factory=list)
packages_paths: List[Path] = dataclasses.field(default_factory=list)
packages_urls: list[str] = dataclasses.field(default_factory=list)
packages_paths: list[Path] = dataclasses.field(default_factory=list)
attempts: int = dataclasses.field(default=0)
def __post_init__(self) -> None:
@ -388,7 +388,7 @@ def get_image(test: Test, client: DockerClient) -> Image:
time.sleep(1)
def join_commands(commands: List[str], sep: str = " && ") -> str:
def join_commands(commands: list[str], sep: str = " && ") -> str:
return sep.join(commands)
@ -421,7 +421,7 @@ def run_test(test: Test, client: DockerClient) -> Result:
test_external_root = Path(__file__).parent.resolve()
logging.debug(test_external_root)
log_external_path = Path.joinpath(test_external_root, log_name)
commands: List[str] = ["export PYTHONIOENCODING=UTF-8"]
commands: list[str] = ["export PYTHONIOENCODING=UTF-8"]
if test.os_name.startswith("rhel"):
# RHEL distros need EPEL for Compass dependencies
@ -552,7 +552,7 @@ r = retry_call(
mongosh_releases = r.json()
def iterate_over_downloads() -> Generator[Dict[str, Any], None, None]:
def iterate_over_downloads() -> Generator[dict[str, Any], None, None]:
# TODO: TOOLS-3204 - we need to sub the arch alias until package
# rchitectures are named consistently with the server packages
for version in current_releases["versions"]:
@ -615,7 +615,7 @@ def get_mongosh_package(arch_name: str, os_name: str) -> Optional[str]:
return None
def get_arch_aliases(arch_name: str) -> List[str]:
def get_arch_aliases(arch_name: str) -> list[str]:
if arch_name in ("amd64", "x86_64"):
return ["amd64", "x86_64"]
if arch_name in ("ppc64le", "ppc64el"):
@ -662,10 +662,10 @@ def validate_no_libdwarf(sources_text, edition, binfile):
raise Exception(f"Found LGPL code from libdwarf in {edition} binary {binfile}.")
arches: Set[str] = set()
oses: Set[str] = set()
editions: Set[str] = set()
versions: Set[str] = set()
arches: set[str] = set()
oses: set[str] = set()
editions: set[str] = set()
versions: set[str] = set()
for dl in iterate_over_downloads():
editions.add(get_edition_alias(dl["edition"]))
@ -765,8 +765,8 @@ arch: str = args.arch
if arch == "auto":
arch = platform.machine()
tests: List[Test] = []
urls: List[str] = []
tests: list[Test] = []
urls: list[str] = []
if args.command == "branch":
# If evg-build-id is provided, download the packages locally using the Evergreen API
@ -916,7 +916,7 @@ if args.command == "release":
continue
test_os: str = dl["target"]
urls: List[str] = dl["packages"]
urls: list[str] = dl["packages"]
server_version: str = dl["version"]
edition: str

22
buildscripts/package_test_internal.py
View File

@ -16,7 +16,7 @@ import tarfile
import time
import traceback
from logging.handlers import WatchedFileHandler
from typing import Dict, List, Optional, Tuple, Union
from typing import Optional, Union
root = logging.getLogger()
root.setLevel(logging.DEBUG)
@ -39,7 +39,7 @@ JOURNALCTL_URL = (
DOCKER_SYSTEMCTL_REPO + "/eb2a963a7d8413119b432bcb6151af6076b65f84/files/docker/journalctl3.py"
)
TestArgs = Dict[str, Union[str, int, List[str]]]
TestArgs = dict[str, Union[str, int, list[str]]]
def run_and_log(cmd: str, end_on_error: bool = True):
@ -53,7 +53,7 @@ def run_and_log(cmd: str, end_on_error: bool = True):
return proc
def download_extract_package(package: str) -> List[str]:
def download_extract_package(package: str) -> list[str]:
# Handle local files (file:// protocol) - these are pre-downloaded by package_test.py
# when using --evg-build-id for private artifacts
if package.startswith("file://"):
@ -82,7 +82,7 @@ def download_extract_package(package: str) -> List[str]:
return extracted_paths
def download_extract_all_packages(package_urls: List[str]) -> List[str]:
def download_extract_all_packages(package_urls: list[str]) -> list[str]:
all_packages = [] # type: List[str]
for package_url in package_urls:
if package_url:
@ -91,17 +91,17 @@ def download_extract_all_packages(package_urls: List[str]) -> List[str]:
return all_packages
def run_apt_test(packages: List[str]):
def run_apt_test(packages: list[str]):
logging.info("Detected apt running test.")
run_and_log("DEBIAN_FRONTEND=noninteractive apt-get install -y {}".format(" ".join(packages)))
def run_yum_test(packages: List[str]):
def run_yum_test(packages: list[str]):
logging.info("Detected yum running test.")
run_and_log("yum install -y {}".format(" ".join(packages)))
def run_zypper_test(packages: List[str]):
def run_zypper_test(packages: list[str]):
logging.info("Detected zypper running test.")
run_and_log("zypper -n --no-gpg-checks install {}".format(" ".join(packages)))
@ -143,7 +143,7 @@ def run_mongo_query(shell, query, should_fail=False, tries=60, interval=1.0):
raise RuntimeError("Query retries exceeded, failing test.")
def parse_os_release(path: str) -> Dict[str, str]:
def parse_os_release(path: str) -> dict[str, str]:
result = {} # type: Dict[str, str]
with open(path, "r", encoding="utf-8") as os_release:
for line in os_release:
@ -156,7 +156,7 @@ def parse_os_release(path: str) -> Dict[str, str]:
return result
def get_os_release() -> Tuple[str, int, int]:
def get_os_release() -> tuple[str, int, int]:
if os.path.exists("/etc/os-release"):
release_info = parse_os_release("/etc/os-release")
elif os.path.exists("/usr/lib/os-release"):
@ -177,7 +177,7 @@ def get_os_release() -> Tuple[str, int, int]:
return os_name, os_version_major, os_version_minor
def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]:
def parse_ulimits(pid: int) -> dict[str, tuple[int, int, Optional[str]]]:
ulimit_line_re = re.compile(
r"(?P<name>.*?)\s{2,}(?P<soft>\S+)\s+(?P<hard>\S+)(?:\s+(?P<units>\S+))?", re.MULTILINE
)
@ -205,7 +205,7 @@ def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]:
return result
def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs:
def get_test_args(package_manager: str, package_files: list[str]) -> TestArgs:
# Set up data for later tests
test_args = {} # type: TestArgs

20
buildscripts/patch_builds/change_data.py
View File

@ -2,14 +2,14 @@
import os
from itertools import chain
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
from typing import Any, Iterable, Optional
import structlog
from git import DiffIndex, Repo
LOGGER = structlog.get_logger(__name__)
RevisionMap = Dict[str, str]
RevisionMap = dict[str, str]
def _get_id_from_repo(repo: Repo) -> str:
@ -24,7 +24,7 @@ def _get_id_from_repo(repo: Repo) -> str:
return os.path.basename(repo.working_dir)
def generate_revision_map(repos: List[Repo], revisions_data: Dict[str, str]) -> RevisionMap:
def generate_revision_map(repos: list[Repo], revisions_data: dict[str, str]) -> RevisionMap:
"""
Generate a revision map for the given repositories using the revisions in the given file.
@ -49,7 +49,7 @@ def _paths_for_iter(diff, iter_type):
return a_path_changes.union(b_path_changes)
def _modified_files_for_diff(diff: DiffIndex, log: Any) -> Set:
def _modified_files_for_diff(diff: DiffIndex, log: Any) -> set:
"""
Get the set of files modified in the given git diff.
@ -72,7 +72,7 @@ def _modified_files_for_diff(diff: DiffIndex, log: Any) -> Set:
return modified_files.union(added_files).union(renamed_files).union(deleted_files)
def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) -> Set[str]:
def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) -> set[str]:
"""
Find files that were new or added to the repository between commits.
@ -104,7 +104,7 @@ def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) -
def find_changed_files_in_repos(
repos: Iterable[Repo], revision_map: Optional[RevisionMap] = None
) -> Set[str]:
) -> set[str]:
"""
Find the changed files.
@ -118,8 +118,8 @@ def find_changed_files_in_repos(
def find_modified_lines_for_files(
repo: Repo, changed_files: List[str], revision_map: Optional[RevisionMap] = None
) -> Dict[str, List[Tuple[int, str]]]:
repo: Repo, changed_files: list[str], revision_map: Optional[RevisionMap] = None
) -> dict[str, list[tuple[int, str]]]:
"""
For each changed file, determine which lines were modified.
@ -164,8 +164,8 @@ def find_modified_lines_for_files(
def find_modified_lines_for_files_in_repos(
repos: Iterable[Repo], changed_files: List[str], revision_map: Optional[RevisionMap] = None
) -> Dict[str, List[Tuple[int, str]]]:
repos: Iterable[Repo], changed_files: list[str], revision_map: Optional[RevisionMap] = None
) -> dict[str, list[tuple[int, str]]]:
"""
Find the modified lines in files with changes.

5
buildscripts/powercycle_sentinel.py
View File

@ -9,7 +9,6 @@ import os
import sys
import time
from datetime import datetime, timezone
from typing import List
import click
import structlog
@ -42,7 +41,7 @@ def get_evergreen_api() -> EvergreenApi:
sys.exit(1)
def watch_tasks(task_ids: List[str], evg_api: EvergreenApi, watch_interval_secs: int) -> List[str]:
def watch_tasks(task_ids: list[str], evg_api: EvergreenApi, watch_interval_secs: int) -> list[str]:
"""Watch tasks if they run longer than exec timeout."""
watch_task_ids = task_ids[:]
long_running_task_ids = []
@ -66,7 +65,7 @@ def watch_tasks(task_ids: List[str], evg_api: EvergreenApi, watch_interval_secs:
return long_running_task_ids
def get_links(task_ids: List[str]) -> str:
def get_links(task_ids: list[str]) -> str:
"""Return evergreen task urls delimited by newline."""
return "\n".join([f"{EVERGREEN_HOST}/task/{task_id}" for task_id in task_ids])

3
buildscripts/pyrightlint.py
View File

@ -6,7 +6,6 @@ import logging
import os
import subprocess
import sys
from typing import List
import structlog
@ -27,7 +26,7 @@ def is_interesting_file(filename: str) -> bool:
return filename.endswith(".py")
def lint(paths: List[str]):
def lint(paths: list[str]):
"""Lint specified paths (files or directories) using Pyright."""
if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
subprocess.run(

9
buildscripts/quickmongolint.py
View File

@ -6,7 +6,6 @@ import logging
import os
import re
import sys
from typing import List
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
@ -33,7 +32,7 @@ def is_interesting_file(file_name: str) -> bool:
) and FILES_RE.search(file_name)
def _lint_files(file_names: List[str]) -> None:
def _lint_files(file_names: list[str]) -> None:
"""Lint a list of files with clang-format."""
def run_lint1(param1):
@ -53,7 +52,7 @@ def lint_patch(file_name: str) -> None:
_lint_files(file_names)
def lint(file_names: List[str]) -> None:
def lint(file_names: list[str]) -> None:
# type: (str, Dict[str, str], List[str]) -> None
"""Lint files command entry point."""
all_file_names = git.get_files_to_check(file_names, is_interesting_file)
@ -61,14 +60,14 @@ def lint(file_names: List[str]) -> None:
_lint_files(all_file_names)
def lint_all(file_names: List[str]) -> None:
def lint_all(file_names: list[str]) -> None:
"""Lint files command entry point based on working tree."""
all_file_names = git.get_files_to_check_working_tree(is_interesting_file)
_lint_files(all_file_names)
def lint_my(origin_branch: List[str]) -> None:
def lint_my(origin_branch: list[str]) -> None:
"""Lint files command based on local changes."""
files = git.get_my_files_to_check(is_interesting_file, origin_branch)
files = [f for f in files if os.path.exists(f)]

6
buildscripts/resmoke_proxy/resmoke_proxy.py
View File

@ -1,6 +1,6 @@
"""A service to proxy requests to resmoke."""
from typing import Any, Dict, List
from typing import Any
import inject
@ -17,7 +17,7 @@ class ResmokeProxyService:
_parser.set_run_options(run_options)
self._suite_config = _suiteconfig
def list_tests(self, suite_name: str) -> List[str]:
def list_tests(self, suite_name: str) -> list[str]:
"""
List the test files that are part of the suite.
@ -35,7 +35,7 @@ class ResmokeProxyService:
return test_list
def read_suite_config(self, suite_name: str) -> Dict[str, Any]:
def read_suite_config(self, suite_name: str) -> dict[str, Any]:
"""
Read the given resmoke suite configuration.

7
buildscripts/resmoke_tests_runtime_validate.py
View File

@ -5,7 +5,6 @@ import json
import sys
from collections import namedtuple
from statistics import mean
from typing import Dict, List
import click
import structlog
@ -26,7 +25,7 @@ IGNORE_LESS_THAN_SECS = 15
_TestData = namedtuple("TestData", ["test_file", "duration"])
def parse_resmoke_report(report_file: str) -> List[TestInfo]:
def parse_resmoke_report(report_file: str) -> list[TestInfo]:
"""Get js tests info from resmoke report json."""
with open(report_file, "r") as fh:
report_data = json.load(fh)
@ -36,7 +35,7 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]:
def get_historic_stats(
project_id: str, task_name: str, build_variant: str
) -> List[HistoricalTestInformation]:
) -> list[HistoricalTestInformation]:
"""Get historic test stats."""
base_task_name = get_task_name_without_suffix(task_name, build_variant).replace(
BURN_IN_PREFIX, ""
@ -44,7 +43,7 @@ def get_historic_stats(
return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant)
def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]:
def make_stats_map(stats: list[_TestData]) -> dict[str, list[float]]:
"""Make test stats map."""
stats_map = {}

13
buildscripts/resmokelib/bazel_suite_parser.py
View File

@ -11,7 +11,6 @@ subset of Bazel syntax:
import functools
import os
import re
from typing import Dict, List
class BazelParseError(Exception):
@ -21,7 +20,7 @@ class BazelParseError(Exception):
@functools.cache
def parse_resmoke_suite_test(target_label: str) -> Dict[str, List[str]]:
def parse_resmoke_suite_test(target_label: str) -> dict[str, list[str]]:
"""Parse a resmoke_suite_test target from BUILD.bazel.
Args:
target_label: Bazel target label like "//buildscripts/resmokeconfig:core"
@ -121,7 +120,7 @@ def _parse_label(target_label: str) -> tuple[str, str]:
return package, target_name
def _parse_load_statements(content: str, package: str) -> Dict[str, str]:
def _parse_load_statements(content: str, package: str) -> dict[str, str]:
"""Parse load statements from BUILD.bazel content.
Extracts identifier to .bzl file mappings from load statements.
@ -172,9 +171,9 @@ def _parse_load_statements(content: str, package: str) -> Dict[str, str]:
def _extract_attribute(
block: str,
attribute_name: str,
identifier_to_bzl_file: Dict[str, str] = None,
identifier_to_bzl_file: dict[str, str] = None,
build_file: str = None,
) -> List[str]:
) -> list[str]:
"""Extract an attribute from a BUILD.bazel rule block.
Supports simple lists and list concatenation with identifiers:
@ -232,8 +231,8 @@ def _extract_attribute(
def _resolve_identifier_to_labels(
identifier: str, identifier_to_bzl_file: Dict[str, str], build_file: str = None
) -> List[str]:
identifier: str, identifier_to_bzl_file: dict[str, str], build_file: str = None
) -> list[str]:
"""Convert a Bazel identifier to a list of labels.
This function resolves identifiers used in list concatenation expressions.

6
buildscripts/resmokelib/configure_resmoke.py
View File

@ -16,7 +16,7 @@ import textwrap
import traceback
from functools import cache
from pathlib import Path
from typing import Dict, List, Optional
from typing import Optional
import git
import pymongo.uri_parser
@ -299,8 +299,8 @@ def _set_up_tracing(
otel_collector_dir: Optional[str],
trace_id: Optional[str],
parent_span_id: Optional[str],
extra_context: Dict[str, object],
suite_files: Optional[List[str]] = None,
extra_context: dict[str, object],
suite_files: Optional[list[str]] = None,
shard_index: Optional[int] = None,
) -> bool:
"""Try to set up otel tracing. On success return True. On failure return False.

3
buildscripts/resmokelib/core/pipe.py
View File

@ -7,7 +7,6 @@ being waited on.
import threading
from textwrap import wrap
from typing import List
# Logkeeper only support log lines up to 4 MB, we want to be a little under that to account for
# extra metadata that gets sent along with the log message.
@ -85,7 +84,7 @@ class LoggerPipe(threading.Thread):
LoggerPipe.__join(self) # Tidy up the started thread.
@staticmethod
def _format_line_for_logging(line_bytes: bytes) -> List[str]:
def _format_line_for_logging(line_bytes: bytes) -> list[str]:
"""
Convert the given byte array into string(s) to be send to the logger.

6
buildscripts/resmokelib/core/programs.py
View File

@ -8,7 +8,7 @@ import os
import os.path
import re
import stat
from typing import Any, Optional, Tuple
from typing import Any, Optional
from opentelemetry import trace
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
@ -186,7 +186,7 @@ def mongos_program(
executable: Optional[str] = None,
process_kwargs: Optional[dict] = None,
mongos_options: dict = None,
) -> Tuple[process.Process, dict]:
) -> tuple[process.Process, dict]:
"""Return a Process instance that starts a mongos with arguments constructed from 'kwargs'."""
bin_version = get_binary_version(executable)
args = [executable]
@ -257,7 +257,7 @@ def mongos_program(
def mongot_program(
logger, job_num, executable=None, process_kwargs=None, mongot_options=None
) -> Tuple[process.Process, Any]:
) -> tuple[process.Process, Any]:
"""Return a Process instance that starts a mongot."""
args = [executable]
mongot_options = mongot_options.copy()

4
buildscripts/resmokelib/discovery/__init__.py
View File

@ -1,7 +1,7 @@
"""Subcommands for test discovery."""
import argparse
from typing import List, Optional
from typing import Optional
import yaml
from pydantic import BaseModel
@ -25,7 +25,7 @@ class SuiteTestList(BaseModel):
"""Collection of tests belonging to a suite."""
suite_name: str
tests: List[str]
tests: list[str]
class TestDiscoverySubcommand(Subcommand):

6
buildscripts/resmokelib/extensions/add_extensions_signature_pub_key_path.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
from typing import Dict, Optional
from typing import Optional
from buildscripts.resmokelib.extensions.constants import (
TEST_PUBLIC_KEY_PATH,
@ -9,8 +9,8 @@ from buildscripts.resmokelib.extensions.constants import (
def add_extensions_signature_pub_key_path(
skip_extensions_signature_verification,
config,
mongod_options: Dict,
mongos_options: Optional[Dict] = None,
mongod_options: dict,
mongos_options: Optional[dict] = None,
):
# We omit providing the extension signature public key path parameter if we intend to skip signature verification.
# This signals to the server in insecure mode to skip validating extension signatures at load time.

10
buildscripts/resmokelib/hang_analyzer/dumper.py
View File

@ -15,7 +15,7 @@ from abc import ABCMeta, abstractmethod
from collections import namedtuple
from datetime import datetime, timedelta
from io import StringIO
from typing import List, Optional, Tuple
from typing import Optional
import psutil
from opentelemetry import trace
@ -73,17 +73,17 @@ def get_dumpers(root_logger: logging.Logger, dbg_output: str, include_terminatin
return Dumpers(dbg=dbg, jstack=jstack)
def find_files(file_name: str, path: str) -> List[str]:
def find_files(file_name: str, path: str) -> list[str]:
matches = glob.glob(f"{path}/**/{file_name}", recursive=True)
return [f for f in matches if os.path.isfile(f)]
def filter_core_dumps(
core_files: List[str],
core_files: list[str],
boring_core_dump_pids: set,
max_core_dumps: int,
logger: logging.Logger,
) -> List[str]:
) -> list[str]:
"""
Filter core dump files by removing boring PIDs and applying a maximum cap.
@ -849,7 +849,7 @@ class GDBDumper(Dumper):
multiversion_dir: str,
logger: logging.Logger,
gdb_index_cache: str,
) -> Tuple[int, str]: # returns (exit_code, test_status)
) -> tuple[int, str]: # returns (exit_code, test_status)
cmds = []
dbg = self.find_debugger()
readelf = find_program("eu-readelf", ["/opt/mongodbtoolchain/v5/bin", "/usr/bin"])

10
buildscripts/resmokelib/hang_analyzer/gen_hang_analyzer_tasks.py
View File

@ -11,7 +11,7 @@ import re
import string
import sys
from abc import ABC, abstractmethod
from typing import List, NamedTuple, Optional
from typing import NamedTuple, Optional
from unittest.mock import MagicMock
from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskDependency
@ -51,7 +51,7 @@ class CoreAnalysisTaskGenerator(ABC):
gdb_index_cache: str,
has_interesting_core_dumps: bool,
boring_core_dump_pids: set,
) -> List[FunctionCall]:
) -> list[FunctionCall]:
pass
@abstractmethod
@ -216,7 +216,7 @@ class ResmokeCoreAnalysisTaskGenerator(CoreAnalysisTaskGenerator):
gdb_index_cache: str,
has_interesting_core_dumps: bool,
boring_core_dump_pids: set,
) -> List[FunctionCall]:
) -> list[FunctionCall]:
return _get_core_analyzer_commands(
task_id,
execution,
@ -279,7 +279,7 @@ class BazelCoreAnalysisTaskGenerator(CoreAnalysisTaskGenerator):
gdb_index_cache: str,
has_interesting_core_dumps: bool,
boring_core_dump_pids: set,
) -> List[FunctionCall]:
) -> list[FunctionCall]:
return _get_core_analyzer_commands(
task_id,
execution,
@ -347,7 +347,7 @@ def _get_core_analyzer_commands(
has_interesting_core_dumps: bool,
boring_core_dump_pids: set,
is_bazel_task: bool = False,
) -> List[FunctionCall]:
) -> list[FunctionCall]:
"""Return setup commands."""
return [
FunctionCall("f_expansions_write"),

7
buildscripts/resmokelib/hang_analyzer/hang_analyzer.py
View File

@ -18,7 +18,6 @@ import re
import signal
import sys
import traceback
from typing import List
import distro
import psutil
@ -41,7 +40,7 @@ class HangAnalyzer(Subcommand):
"""
self.options = options
self.root_logger = None
self.interesting_processes: List[str] = [
self.interesting_processes: list[str] = [
# Remove "python", "java" from the list to avoid hang analyzer multiple invocations
"mongo",
"mongod",
@ -49,8 +48,8 @@ class HangAnalyzer(Subcommand):
"_test",
"dbtest",
]
self.go_processes: List[str] = []
self.process_ids: List[int] = []
self.go_processes: list[str] = []
self.process_ids: list[int] = []
def configure_task_id():
run_tid = resmoke_config.EVERGREEN_TASK_ID

4
buildscripts/resmokelib/hang_analyzer/process_list.py
View File

@ -4,7 +4,7 @@ import csv
import io
import os
import sys
from typing import List, NamedTuple, Union
from typing import NamedTuple, Union
from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program
@ -13,7 +13,7 @@ class Pinfo(NamedTuple):
"""Holds a vector of PIDs of the same process type."""
name: str
pidv: Union[int, List[int]]
pidv: Union[int, list[int]]
def get_processes(process_ids, interesting_processes, process_match, logger):

4
buildscripts/resmokelib/multiversion/__init__.py
View File

@ -1,7 +1,7 @@
"""Subcommand for multiversion config."""
import argparse
from typing import List, Optional
from typing import Optional
import yaml
from pydantic import BaseModel
@ -30,7 +30,7 @@ class MultiversionConfig(BaseModel):
* last_continuous_fcv: Continuous version that should be tested against.
"""
last_versions: List[str]
last_versions: list[str]
requires_fcv_tag: str
requires_fcv_tag_lts: str
requires_fcv_tag_continuous: str

26
buildscripts/resmokelib/multiversion/multiversion_service.py
View File

@ -4,7 +4,7 @@ from __future__ import annotations
import re
from bisect import bisect_left, bisect_right
from typing import List, NamedTuple, Optional
from typing import NamedTuple, Optional
import structlog
import yaml
@ -42,10 +42,10 @@ class VersionConstantValues(NamedTuple):
latest: Version
last_continuous: Version
last_lts: Version
requires_fcv_tag_list: List[Version]
requires_fcv_tag_list_continuous: List[Version]
fcvs_less_than_latest: List[Version]
eols: List[Version]
requires_fcv_tag_list: list[Version]
requires_fcv_tag_list_continuous: list[Version]
fcvs_less_than_latest: list[Version]
eols: list[Version]
def get_fcv_tag_list(self) -> str:
"""Get a comma joined string of all the fcv tags."""
@ -78,7 +78,7 @@ class VersionConstantValues(NamedTuple):
"""Get a string version of the latest FCV."""
return version_str(self.latest)
def get_fcv_tags_less_than_latest(self) -> List[str]:
def get_fcv_tags_less_than_latest(self) -> list[str]:
"""Get the list of all fcv tags less than the latest."""
return [tag_str(fcv) for fcv in self.fcvs_less_than_latest]
@ -102,7 +102,7 @@ class VersionConstantValues(NamedTuple):
last_continuous = self.get_last_continuous_fcv()
return f"{base_name}-{last_continuous}"
def get_eols(self) -> List[str]:
def get_eols(self) -> list[str]:
"""Get EOL'd versions as list of strings."""
return [version_str(eol) for eol in self.eols]
@ -148,9 +148,9 @@ class MongoReleases(BaseModel):
LTS.
"""
feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions")
long_term_support_releases: List[str] = Field(alias="longTermSupportReleases")
eol_versions: List[str] = Field(alias="eolVersions")
feature_compatibility_versions: list[str] = Field(alias="featureCompatibilityVersions")
long_term_support_releases: list[str] = Field(alias="longTermSupportReleases")
eol_versions: list[str] = Field(alias="eolVersions")
generate_fcv_lower_bound_override: Optional[str] = Field(
None, alias="generateFCVLowerBoundOverride"
)
@ -178,15 +178,15 @@ class MongoReleases(BaseModel):
)
raise
def get_fcv_versions(self) -> List[Version]:
def get_fcv_versions(self) -> list[Version]:
"""Get the Version representation of all fcv versions."""
return [Version(fcv) for fcv in self.feature_compatibility_versions]
def get_lts_versions(self) -> List[Version]:
def get_lts_versions(self) -> list[Version]:
"""Get the Version representation of the lts versions."""
return [Version(lts) for lts in self.long_term_support_releases]
def get_eol_versions(self) -> List[Version]:
def get_eol_versions(self) -> list[Version]:
"""Get the Version representation of the EOL versions."""
return [Version(eol) for eol in self.eol_versions]

16
buildscripts/resmokelib/run/__init__.py
View File

@ -14,7 +14,7 @@ import textwrap
import time
from abc import ABC, abstractmethod
from logging import Logger
from typing import List, Optional
from typing import Optional
import psutil
import yaml
@ -173,7 +173,7 @@ class TestRunner(Subcommand):
)
@staticmethod
def _find_suites_by_test(suites: List[Suite]):
def _find_suites_by_test(suites: list[Suite]):
"""
Look up what other resmoke suites run the tests specified in the suites parameter.
@ -1016,7 +1016,7 @@ class TestRunner(Subcommand):
)
suite.tests = config.SHUFFLE_STRATEGY.shuffle(suite.tests)
def _get_suites(self) -> List[Suite]:
def _get_suites(self) -> list[Suite]:
"""Return the list of suites for this resmoke invocation."""
try:
return suitesconfig.get_suites(config.SUITE_FILES, config.TEST_FILES)
@ -1057,11 +1057,11 @@ class TestRunner(Subcommand):
@staticmethod
def _get_suite_summary(suite: Suite):
"""Return a summary of the suite run."""
sb: List[str] = []
sb: list[str] = []
suite.summarize(sb)
return "\n".join(sb)
def _setup_signal_handler(self, suites: List[Suite]):
def _setup_signal_handler(self, suites: list[Suite]):
"""Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump."""
sighandler.register(self._resmoke_logger, suites, self.__start_time)
@ -1166,7 +1166,7 @@ class TestRunnerEvg(TestRunner):
return combinations
def _get_suites(self) -> List[Suite]:
def _get_suites(self) -> list[Suite]:
"""Return a list of resmokelib.testing.suite.Suite instances to execute.
For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(),
@ -2549,7 +2549,7 @@ class RunPlugin(PluginInterface):
def to_local_args(
input_args: Optional[List[str]] = None, additional_skipped_args: Optional[List[str]] = None
input_args: Optional[list[str]] = None, additional_skipped_args: Optional[list[str]] = None
):
"""
Return a command line invocation for resmoke.py suitable for being run outside of Evergreen.
@ -2669,7 +2669,7 @@ def to_local_args(
)
def strip_fuzz_config_params(input_args: List[str]):
def strip_fuzz_config_params(input_args: list[str]):
"""Delete fuzz related command line args because we have to add the seed manually."""
ret = []

20
buildscripts/resmokelib/selector.py
View File

@ -11,7 +11,7 @@ import os.path
import random
import subprocess
import sys
from typing import Any, List, NamedTuple
from typing import Any, NamedTuple
import buildscripts.resmokelib.testing.tags as _tags
from buildscripts.resmokelib import config, errors, utils
@ -168,8 +168,8 @@ class TestFileExplorer(object):
class _EvaluatePathsResult(NamedTuple):
"""Results of paths evaluation."""
evaluated: List[str]
unrecognized: List[str]
evaluated: list[str]
unrecognized: list[str]
class _TestList(object):
@ -185,7 +185,7 @@ class _TestList(object):
"""
def __init__(
self, test_file_explorer: TestFileExplorer, roots: List[str], tests_are_files: bool = True
self, test_file_explorer: TestFileExplorer, roots: list[str], tests_are_files: bool = True
) -> None:
"""Initialize the _TestList with a TestFileExplorer component and a list of root tests."""
self._test_file_explorer = test_file_explorer
@ -193,7 +193,7 @@ class _TestList(object):
self._roots = self._expand_roots(roots) if tests_are_files else roots
self._filtered = set(self._roots)
def _evaluate_paths(self, paths: List[str]) -> _EvaluatePathsResult:
def _evaluate_paths(self, paths: list[str]) -> _EvaluatePathsResult:
evaluated = []
unrecognized = []
@ -217,7 +217,7 @@ class _TestList(object):
return _EvaluatePathsResult(evaluated, unrecognized)
def _expand_roots(self, tests: List[str]) -> List[str]:
def _expand_roots(self, tests: list[str]) -> list[str]:
paths = self._evaluate_paths(tests)
if len(paths.unrecognized) > 0 and config.VALIDATE_SELECTOR_PATHS:
raise errors.SuiteSelectorConfigurationError(
@ -227,7 +227,7 @@ class _TestList(object):
)
return paths.evaluated
def include_files(self, include_files: List[str]) -> None:
def include_files(self, include_files: list[str]) -> None:
"""
Filter the test list so that it only includes files matching 'include_files'.
@ -259,7 +259,7 @@ class _TestList(object):
self._filtered = set(paths.evaluated)
def exclude_files(self, exclude_files: List[str]) -> None:
def exclude_files(self, exclude_files: list[str]) -> None:
"""
Exclude from the test list the files that match elements from 'exclude_files'.
@ -912,7 +912,7 @@ _SELECTOR_REGISTRY = {
def filter_tests(
test_kind, selector_config, test_file_explorer=_DEFAULT_TEST_FILE_EXPLORER
) -> tuple[List[str], List[str]]:
) -> tuple[list[str], list[str]]:
"""Filter the tests according to a specified configuration.
Args:
@ -931,7 +931,7 @@ def filter_tests(
def group_tests(
test_kind, selector_config, test_list, test_file_explorer=_DEFAULT_TEST_FILE_EXPLORER
) -> List[Any]:
) -> list[Any]:
"""Group the test list according to a specified configuration.
Args:

8
buildscripts/resmokelib/setup_multiversion/config.py
View File

@ -1,7 +1,5 @@
"""Setup multiversion config."""
from typing import List
SETUP_MULTIVERSION_CONFIG = (
"buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml"
)
@ -17,7 +15,7 @@ class Buildvariant:
edition: str
platform: str
architecture: str
versions: List[str]
versions: list[str]
def __init__(self, buildvariant_yaml: dict):
"""Initialize."""
@ -31,8 +29,8 @@ class Buildvariant:
class SetupMultiversionConfig:
"""Class represents setup multiversion config."""
evergreen_projects: List[str]
evergreen_buildvariants: List[Buildvariant]
evergreen_projects: list[str]
evergreen_buildvariants: list[Buildvariant]
def __init__(self, raw_yaml: dict):
"""Initialize."""

8
buildscripts/resmokelib/setup_multiversion/setup_multiversion.py
View File

@ -15,7 +15,7 @@ import subprocess
import sys
import time
from itertools import chain
from typing import Any, Dict, List, NamedTuple, Optional
from typing import Any, NamedTuple, Optional
import distro
import yaml
@ -71,7 +71,7 @@ def get_merge_base_commit(version: str, logger: logging.Logger) -> Optional[str]
class EvgURLInfo(NamedTuple):
"""Wrapper around compile URLs with metadata."""
urls: Dict[str, Any] = {}
urls: dict[str, Any] = {}
evg_version_id: str = None
@ -173,7 +173,7 @@ class SetupMultiversion(Subcommand):
def _get_release_versions(
self, install_last_lts: Optional[bool], install_last_continuous: Optional[bool]
) -> List[str]:
) -> list[str]:
"""Return last-LTS and/or last-continuous versions."""
out = []
if not os.path.isfile(
@ -338,7 +338,7 @@ class SetupMultiversion(Subcommand):
"Finished writing binary paths on Windows to %s", config.WINDOWS_BIN_PATHS_FILE
)
def _write_evg_versions_file(self, file_name: str, versions: List[str]):
def _write_evg_versions_file(self, file_name: str, versions: list[str]):
with open(file_name, "a") as out:
out.write("\n".join(versions))

8
buildscripts/resmokelib/testing/executor.py
View File

@ -2,7 +2,7 @@
import threading
from logging import Logger
from typing import Generic, List, Optional, TypeVar, Union
from typing import Generic, Optional, TypeVar, Union
from opentelemetry import context
@ -71,7 +71,7 @@ class TestSuiteExecutor(object):
self._jobs = self._create_jobs(suite.get_num_jobs_to_start())
def _create_jobs(self, num_jobs: int) -> List[_job.Job]:
def _create_jobs(self, num_jobs: int) -> list[_job.Job]:
"""
Start jobs.
@ -272,7 +272,7 @@ class TestSuiteExecutor(object):
return fixtures.make_fixture(fixture_class, fixture_logger, job_num, **fixture_config)
def _make_hooks(self, fixture, job_num) -> List[Hook]:
def _make_hooks(self, fixture, job_num) -> list[Hook]:
"""Create the hooks for the job's fixture."""
hooks = []
@ -376,7 +376,7 @@ class TestQueue(_queue.Queue, Generic[T]):
self.max_test_queue_size = utils.default_if_none(_config.MAX_TEST_QUEUE_SIZE, -1)
super().__init__()
def add_test_cases(self, test_cases: List[QueueElem]) -> None:
def add_test_cases(self, test_cases: list[QueueElem]) -> None:
"""Add test cases to the queue."""
for test_case in test_cases:
if self.max_test_queue_size < 0 or self.num_tests < self.max_test_queue_size:

38
buildscripts/resmokelib/testing/fixtures/_builder.py
View File

@ -4,7 +4,7 @@ import json
import logging
import threading
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Tuple, Type
from typing import Any, Optional
from buildscripts.resmokelib import config, errors, logging
from buildscripts.resmokelib.suitesconfig import _get_suite_config
@ -143,8 +143,8 @@ class FixtureContainer(object):
def _extract_multiversion_options(
kwargs: Dict[str, Any],
) -> Tuple[Optional[List[str]], Optional[str]]:
kwargs: dict[str, Any],
) -> tuple[Optional[list[str]], Optional[str]]:
"""Pop multiversion options from kwargs dict and return them.
:param kwargs: fixture kwargs
@ -176,9 +176,9 @@ class ReplSetBuilder(FixtureBuilder):
self,
logger: logging.Logger,
job_num: int,
fixturelib: Type[FixtureLib],
fixturelib: type[FixtureLib],
*args,
existing_nodes: Optional[List[MongoDFixture]] = None,
existing_nodes: Optional[list[MongoDFixture]] = None,
**kwargs,
) -> ReplicaSetFixture:
"""Build a replica set.
@ -269,7 +269,7 @@ class ReplSetBuilder(FixtureBuilder):
return replset
@staticmethod
def _mutate_kwargs(kwargs: Dict[str, Any]) -> None:
def _mutate_kwargs(kwargs: dict[str, Any]) -> None:
"""Update replica set fixture kwargs.
:param kwargs: replica set fixture kwargs
@ -287,7 +287,7 @@ class ReplSetBuilder(FixtureBuilder):
@staticmethod
def _validate_multiversion_options(
kwargs: Dict[str, Any], mixed_bin_versions: Optional[List[str]]
kwargs: dict[str, Any], mixed_bin_versions: Optional[list[str]]
) -> None:
"""Error out if the number of binary versions does not match the number of nodes in replica set.
@ -311,10 +311,10 @@ class ReplSetBuilder(FixtureBuilder):
@classmethod
def _get_mongod_assets(
cls,
kwargs: Dict[str, Any],
mixed_bin_versions: Optional[List[str]],
kwargs: dict[str, Any],
mixed_bin_versions: Optional[list[str]],
old_bin_version: Optional[str],
) -> Tuple[Dict[str, str], Dict[str, str], List[str]]:
) -> tuple[dict[str, str], dict[str, str], list[str]]:
"""Make dicts with mongod new/old class and executable names and binary versions.
:param kwargs: sharded cluster fixture kwargs
@ -365,7 +365,7 @@ class ReplSetBuilder(FixtureBuilder):
def _new_mongod(
replset: ReplicaSetFixture,
replset_node_index: int,
executables: Dict[str, str],
executables: dict[str, str],
_class: str,
cur_version: str,
is_multiversion: bool,
@ -442,7 +442,7 @@ class ShardedClusterBuilder(FixtureBuilder):
LATEST_MONGOS_CLASS = "_MongoSFixture"
def build_fixture(
self, logger: logging.Logger, job_num: int, fixturelib: Type[FixtureLib], *args, **kwargs
self, logger: logging.Logger, job_num: int, fixturelib: type[FixtureLib], *args, **kwargs
) -> ShardedClusterFixture:
"""Build a sharded cluster.
@ -546,7 +546,7 @@ class ShardedClusterBuilder(FixtureBuilder):
return sharded_cluster
@staticmethod
def _mutate_kwargs(kwargs: Dict[str, Any]) -> None:
def _mutate_kwargs(kwargs: dict[str, Any]) -> None:
"""Update sharded cluster fixture kwargs.
:param kwargs: sharded cluster fixture kwargs
@ -585,7 +585,7 @@ class ShardedClusterBuilder(FixtureBuilder):
@staticmethod
def _validate_multiversion_options(
kwargs: Dict[str, Any], mixed_bin_versions: Optional[List[str]]
kwargs: dict[str, Any], mixed_bin_versions: Optional[list[str]]
) -> None:
"""Error out if the number of binary versions does not match the number of nodes in sharded cluster.
@ -606,10 +606,10 @@ class ShardedClusterBuilder(FixtureBuilder):
@classmethod
def _get_mongos_assets(
cls,
kwargs: Dict[str, Any],
mixed_bin_versions: Optional[List[str]],
kwargs: dict[str, Any],
mixed_bin_versions: Optional[list[str]],
old_bin_version: Optional[str],
) -> Tuple[Dict[str, str], Dict[str, str]]:
) -> tuple[dict[str, str], dict[str, str]]:
"""Make dicts with mongos new/old class and executable names.
:param kwargs: sharded cluster fixture kwargs
@ -670,7 +670,7 @@ class ShardedClusterBuilder(FixtureBuilder):
@staticmethod
def _new_rs_shard(
sharded_cluster: ShardedClusterFixture,
mixed_bin_versions: Optional[List[str]],
mixed_bin_versions: Optional[list[str]],
old_bin_version: Optional[str],
rs_shard_index: int,
num_rs_nodes_per_shard: int,
@ -713,7 +713,7 @@ class ShardedClusterBuilder(FixtureBuilder):
@staticmethod
def _new_mongos(
sharded_cluster: ShardedClusterFixture,
executables: Dict[str, str],
executables: dict[str, str],
_class: str,
mongos_index: int,
total: int,

3
buildscripts/resmokelib/testing/fixtures/fixturelib.py
View File

@ -1,7 +1,6 @@
"""Facade wrapping the resmokelib dependencies used by fixtures."""
from logging import Handler, Logger
from typing import Dict
from buildscripts.resmokelib import config, core, errors, logging, utils
from buildscripts.resmokelib.core import network
@ -121,7 +120,7 @@ class FixtureLib:
SET_PARAMETERS_KEY = "set_parameters"
def merge_mongo_option_dicts(self, original: Dict, override: Dict):
def merge_mongo_option_dicts(self, original: dict, override: dict):
"""
Merge mongod/s options such that --setParameter is merged recursively.

16
buildscripts/resmokelib/testing/fixtures/interface.py
View File

@ -6,7 +6,7 @@ from collections import namedtuple
from enum import Enum
from logging import Logger
from threading import Lock
from typing import TYPE_CHECKING, List
from typing import TYPE_CHECKING
import pymongo
import pymongo.errors
@ -251,11 +251,11 @@ class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):
def __repr__(self):
return "%r(%r, %r)" % (self.__class__.__name__, self.logger, self.job_num)
def get_independent_clusters(self) -> List["Fixture"]:
def get_independent_clusters(self) -> list["Fixture"]:
"""Return a list of the independent clusters that participate in this fixture."""
return [self]
def get_testable_clusters(self) -> List["Fixture"]:
def get_testable_clusters(self) -> list["Fixture"]:
"""Return a list of the clusters in this fixture that we want to run tests against."""
return [self]
@ -274,7 +274,7 @@ class _DockerComposeInterface:
by leveraging the `all_processes` method to access the startup args.
"""
def _all_mongo_d_s_t(self) -> List[Fixture]:
def _all_mongo_d_s_t(self) -> list[Fixture]:
"""
Return a list of all mongo{d,s,t} `Fixture` instances in this fixture.
@ -284,7 +284,7 @@ class _DockerComposeInterface:
"_all_mongo_d_s_t_instances must be implemented by Fixture subclasses that support `docker-compose.yml` generation."
)
def all_processes(self) -> List["Process"]:
def all_processes(self) -> list["Process"]:
"""
Return a list of all `mongo{d,s,t}` `Process` instances in the fixture.
@ -324,13 +324,13 @@ class MultiClusterFixture(Fixture):
REGISTERED_NAME = registry.LEAVE_UNREGISTERED # type: ignore
def get_independent_clusters(self) -> List[Fixture]:
def get_independent_clusters(self) -> list[Fixture]:
"""Return a list of the independent clusters that participate in this fixture."""
raise NotImplementedError(
"get_independent_clusters must be implemented by MultiClusterFixture subclasses"
)
def get_testable_clusters(self) -> List[Fixture]:
def get_testable_clusters(self) -> list[Fixture]:
"""Return a list of the clusters in this fixture that we want to run tests against."""
return self.get_independent_clusters()
@ -474,7 +474,7 @@ class FixtureTeardownHandler(object):
def create_fixture_table(fixture):
"""Get fixture node info, make it a pretty table. Return it or None if fixture is invalid target."""
info: List[NodeInfo] = fixture.get_node_info()
info: list[NodeInfo] = fixture.get_node_info()
if not info:
return None

38
buildscripts/resmokelib/testing/hooks/generate_and_check_perf_results.py
View File

@ -6,7 +6,7 @@ import json
import subprocess
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional
from typing import Any, Optional
import requests
import tenacity
@ -86,8 +86,8 @@ class GenerateAndCheckPerfResults(interface.Hook):
interface.Hook.__init__(self, hook_logger, fixture, GenerateAndCheckPerfResults.DESCRIPTION)
self.cedar_report_file = _config.CEDAR_REPORT_FILE
self.variant = _config.EVERGREEN_VARIANT_NAME
self.cedar_reports: List[CedarTestReport] = []
self.performance_thresholds: Dict[str, Any] = {}
self.cedar_reports: list[CedarTestReport] = []
self.performance_thresholds: dict[str, Any] = {}
self.create_time = None
self.end_time = None
@ -117,8 +117,8 @@ class GenerateAndCheckPerfResults(interface.Hook):
def _check_pass_fail(
self,
benchmark_reports: Dict[str, "_BenchmarkThreadsReport"],
cedar_formatted_results: List[CedarTestReport],
benchmark_reports: dict[str, "_BenchmarkThreadsReport"],
cedar_formatted_results: list[CedarTestReport],
test,
test_report,
):
@ -153,7 +153,7 @@ class GenerateAndCheckPerfResults(interface.Hook):
)
continue
# Transform the thresholds set into something we can more easily use.
metrics_to_check: List[IndividualMetricThreshold] = []
metrics_to_check: list[IndividualMetricThreshold] = []
for item in test_thresholds:
thread_level = item["thread_level"]
for metric in item["metrics"]:
@ -184,7 +184,7 @@ class GenerateAndCheckPerfResults(interface.Hook):
)
)
# Transform the reported performance results into something we can more easily use.
transformed_metrics: Dict[ReportedMetric, CedarMetric] = {}
transformed_metrics: dict[ReportedMetric, CedarMetric] = {}
for cedar_result in cedar_formatted_results:
for individual_metric in cedar_result.metrics:
reported_metric = ReportedMetric(
@ -244,8 +244,8 @@ class GenerateAndCheckPerfResults(interface.Hook):
)
def _generate_cedar_report(
self, benchmark_reports: Dict[str, "_BenchmarkThreadsReport"]
) -> List[CedarTestReport]:
self, benchmark_reports: dict[str, "_BenchmarkThreadsReport"]
) -> list[CedarTestReport]:
"""Format the data to look like a cedar report."""
cedar_report = []
@ -264,10 +264,10 @@ class GenerateAndCheckPerfResults(interface.Hook):
return cedar_report
def _parse_report(self, report_dict) -> Dict[str, "_BenchmarkThreadsReport"]:
def _parse_report(self, report_dict) -> dict[str, "_BenchmarkThreadsReport"]:
context = report_dict["context"]
# Reports grouped by name without thread.
benchmark_reports: Dict[str, "_BenchmarkThreadsReport"] = {}
benchmark_reports: dict[str, "_BenchmarkThreadsReport"] = {}
for benchmark_res in report_dict["benchmarks"]:
bm_name_obj = _BenchmarkThreadsReport.parse_bm_name(benchmark_res)
@ -284,7 +284,7 @@ class GenerateAndCheckPerfResults(interface.Hook):
task_name: str,
variant: str,
measurement: str,
args: Dict[str, Any],
args: dict[str, Any],
base_commit: str,
project: str,
) -> Optional[int]:
@ -360,12 +360,12 @@ class CheckPerfResultTestCase(interface.DynamicTestCase):
description,
base_test_name,
hook,
thresholds_to_check: List["IndividualMetricThreshold"],
reported_metrics: Dict[ReportedMetric, CedarMetric],
thresholds_to_check: list["IndividualMetricThreshold"],
reported_metrics: dict[ReportedMetric, CedarMetric],
):
super().__init__(logger, test_name, description, base_test_name, hook)
self.thresholds_to_check: List["IndividualMetricThreshold"] = thresholds_to_check
self.reported_metrics: Dict[ReportedMetric, CedarMetric] = reported_metrics
self.thresholds_to_check: list["IndividualMetricThreshold"] = thresholds_to_check
self.reported_metrics: dict[ReportedMetric, CedarMetric] = reported_metrics
self.github: Github = Github(get_expansion("github_token_mongo"))
def run_test(self):
@ -577,7 +577,7 @@ class _BenchmarkThreadsReport(object):
"""Add to report."""
self.thread_benchmark_map[bm_name_obj.thread_count].append(report)
def generate_cedar_metrics(self) -> Dict[int, List[CedarMetric]]:
def generate_cedar_metrics(self) -> dict[int, list[CedarMetric]]:
"""
Generate metrics for Cedar.
@ -714,7 +714,7 @@ class _BenchmarkThreadsReport(object):
return res
@staticmethod
def check_dup_metric_names(metrics: List[CedarMetric]) -> bool:
def check_dup_metric_names(metrics: list[CedarMetric]) -> bool:
"""Check duplicated metric names for Cedar."""
names = []
for metric in metrics:
@ -724,7 +724,7 @@ class _BenchmarkThreadsReport(object):
return False
@staticmethod
def parse_bm_name(benchmark_res: Dict[str, Any]):
def parse_bm_name(benchmark_res: dict[str, Any]):
"""
Split the benchmark name into base_name, thread_count and statistic_type.

4
buildscripts/resmokelib/testing/job.py
View File

@ -5,7 +5,7 @@ import sys
import threading
import time
from collections import namedtuple
from typing import TYPE_CHECKING, List, Optional, Union
from typing import TYPE_CHECKING, Optional, Union
from opentelemetry import context, trace
from opentelemetry.context.context import Context
@ -41,7 +41,7 @@ class Job(object):
job_num: int,
logger: logging.Logger,
fixture: Fixture,
hooks: List[Hook],
hooks: list[Hook],
report: TestReport,
archival: HookTestArchival,
suite_options: config.SuiteOptions,

24
buildscripts/resmokelib/testing/suite.py
View File

@ -7,7 +7,7 @@ import threading
import time
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib import selector as _selector
@ -123,7 +123,7 @@ class Suite(object):
self._tests, self._excluded = self._get_tests_for_kind(self.test_kind)
return self._excluded
def _get_tests_for_kind(self, test_kind) -> tuple[List[any], List[str]]:
def _get_tests_for_kind(self, test_kind) -> tuple[list[any], list[str]]:
"""Return the tests to run and those that were excluded, based on the 'test_kind'-specific filtering policy."""
selector_config = self.get_selector_config()
@ -376,7 +376,7 @@ class Suite(object):
return self._reports
@synchronized
def summarize(self, sb: List[str]):
def summarize(self, sb: list[str]):
"""Append a summary of the suite onto the string builder 'sb'."""
if not self._reports and not self._partial_reports:
sb.append("No tests ran.")
@ -585,7 +585,7 @@ class Suite(object):
RETURN_STATUS = "suite_return_status"
ERRORNO = "suite_errorno"
def get_suite_otel_attributes(self) -> Dict[str, Any]:
def get_suite_otel_attributes(self) -> dict[str, Any]:
attributes = {
Suite.METRIC_NAMES.DISPLAY_NAME: self.get_display_name(),
Suite.METRIC_NAMES.NAME: self.get_name(),
@ -602,8 +602,8 @@ class Suite(object):
@staticmethod
def filter_tests_for_shard(
tests: List[str], shard_count: Optional[int], shard_index: Optional[int]
) -> List[str]:
tests: list[str], shard_count: Optional[int], shard_index: Optional[int]
) -> list[str]:
"""Filter tests to only include those that should be run by this shard."""
if shard_index is None or shard_count is None:
return tests
@ -625,15 +625,15 @@ class Suite(object):
class ShardingStrategy(ABC):
@abstractmethod
def get_tests_for_shard(
self, tests: List[str], shard_count: int, shard_index: int
) -> List[str]:
self, tests: list[str], shard_count: int, shard_index: int
) -> list[str]:
pass
class EqualTestCount(ShardingStrategy):
def get_tests_for_shard(
self, tests: List[str], shard_count: int, shard_index: int
) -> List[str]:
self, tests: list[str], shard_count: int, shard_index: int
) -> list[str]:
return [test_case for i, test_case in enumerate(tests) if i % shard_count == shard_index]
@ -644,8 +644,8 @@ class EqualRuntime(ShardingStrategy):
self.runtimes[runtime["test_name"]] = runtime["avg_duration_pass"]
def get_tests_for_shard(
self, tests: List[str], shard_count: int, shard_index: int
) -> List[str]:
self, tests: list[str], shard_count: int, shard_index: int
) -> list[str]:
shards = [[] for _ in range(shard_count)]
shard_runtimes = [0] * shard_count

22
buildscripts/resmokelib/testing/symbolizer_service.py
View File

@ -10,7 +10,7 @@ import sys
import time
from datetime import timedelta
from threading import Lock
from typing import Any, List, NamedTuple, Optional, Set
from typing import Any, NamedTuple, Optional
from opentelemetry import trace
@ -224,7 +224,7 @@ class ResmokeSymbolizer:
def symbolize_stacktraces(
self,
test: TestCase,
files: List[str],
files: list[str],
symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS,
) -> None:
"""
@ -259,7 +259,7 @@ class ResmokeSymbolizer:
def create_unsymbolized_instructions(
self,
test: TestCase,
unsymbolized_stacktraces_info: List[dict],
unsymbolized_stacktraces_info: list[dict],
expansions_file: str = "../expansions.yml",
):
"""
@ -409,7 +409,7 @@ If no symbolized stacktrace is created, then most likely either:
return dbpath
def collect_stacktrace_files(self, dir_path: str) -> List[str]:
def collect_stacktrace_files(self, dir_path: str) -> list[str]:
"""
Collect all stacktrace files which are not empty and return their full paths.
@ -434,7 +434,7 @@ class FileService:
self._processed_files = self.load_processed_files(processed_files_list_path)
@staticmethod
def load_processed_files(file_path: str) -> Set[str]:
def load_processed_files(file_path: str) -> set[str]:
"""
Load processed files info from a file.
@ -446,7 +446,7 @@ class FileService:
return {line for line in set(file.readlines()) if line}
return set()
def add_to_processed_files(self, files: List[str]) -> None:
def add_to_processed_files(self, files: list[str]) -> None:
"""
Bulk add to collection of processed files.
@ -477,7 +477,7 @@ class FileService:
return file in self._processed_files
@staticmethod
def find_all_children_recursively(dir_path: str) -> List[str]:
def find_all_children_recursively(dir_path: str) -> list[str]:
"""
Find all children files in directory recursively.
@ -490,7 +490,7 @@ class FileService:
return children_in_dir
@staticmethod
def filter_by_extension(files: List[str], extension: str) -> List[str]:
def filter_by_extension(files: list[str], extension: str) -> list[str]:
"""
Filter files by extension.
@ -501,7 +501,7 @@ class FileService:
return [f for f in files if f.endswith(extension)]
@staticmethod
def filter_out_non_files(files: List[str]) -> List[str]:
def filter_out_non_files(files: list[str]) -> list[str]:
"""
Filter out non files.
@ -510,7 +510,7 @@ class FileService:
"""
return [f for f in files if os.path.isfile(f)]
def filter_out_already_processed_files(self, files: List[str]):
def filter_out_already_processed_files(self, files: list[str]):
"""
Filter out already processed files.
@ -520,7 +520,7 @@ class FileService:
return [f for f in files if not self.is_processed(f)]
@staticmethod
def filter_out_empty_files(files: List[str]) -> List[str]:
def filter_out_empty_files(files: list[str]) -> list[str]:
"""
Filter our files that are empty.

6
buildscripts/resmokelib/testing/testcases/interface.py
View File

@ -10,7 +10,7 @@ import threading
import timeit
import unittest
import uuid
from typing import Any, Callable, Dict, Optional
from typing import Any, Callable, Optional
import psutil
@ -19,7 +19,7 @@ from buildscripts.resmokelib.hang_analyzer.hang_analyzer import HangAnalyzer
from buildscripts.resmokelib.utils import registry
from buildscripts.resmokelib.utils.self_test_fakes import test_analysis
_TEST_CASES: Dict[str, Callable] = {} # type: ignore
_TEST_CASES: dict[str, Callable] = {} # type: ignore
def make_test_case(test_kind, *args, **kwargs) -> "TestCase":
@ -138,7 +138,7 @@ class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TE
DYNAMIC = "test_dynamic"
BACKGROUND = "test_background"
def get_test_otel_attributes(self) -> Dict[str, Any]:
def get_test_otel_attributes(self) -> dict[str, Any]:
return {
TestCase.METRIC_NAMES.BASE_NAME: self.basename(),
TestCase.METRIC_NAMES.LONG_NAME: self.long_name(),

13
buildscripts/resmokelib/utils/archival.py
View File

@ -12,7 +12,6 @@ import threading
import time
from abc import ABC, abstractmethod
from pathlib import Path
from typing import List
from buildscripts.resmokelib import config
@ -107,13 +106,13 @@ def get_archive_path(path: str) -> str:
class ArchiveStrategy(ABC):
@abstractmethod
def archive_files(self, files: List[str], archive_name: str, display_name: str):
def archive_files(self, files: list[str], archive_name: str, display_name: str):
pass
def exit(self):
pass
def create_tar_archive(self, files: List[str], archive: Path, display_name: str):
def create_tar_archive(self, files: list[str], archive: Path, display_name: str):
status = 0
size_mb = 0
message = "Tar/gzip {} files: {}".format(display_name, files)
@ -284,7 +283,7 @@ class ArchiveToS3(ArchiveStrategy):
self._archive_file_worker.join(timeout=timeout)
self.check_thread(self._archive_file_worker, False)
def archive_files(self, files: List[str], archive_name: str, display_name: str):
def archive_files(self, files: list[str], archive_name: str, display_name: str):
_, temp_file = tempfile.mkstemp(suffix=".tgz")
status, message, size_mb = self.create_tar_archive(files, temp_file, display_name)
self._upload_queue.put(
@ -307,7 +306,7 @@ class ArchiveToDirectory(ArchiveStrategy):
self.logger = logger
os.makedirs(self.directory, exist_ok=True)
def archive_files(self, files: List[str], archive_name: str, display_name: str):
def archive_files(self, files: list[str], archive_name: str, display_name: str):
archive = os.path.join(self.directory, archive_name)
with open(archive, "w") as _:
pass # Touch create the archive file, so we can check the size of the disk it is on before adding to it.
@ -319,7 +318,7 @@ class TestArchival(ArchiveStrategy):
self.archive_file = os.path.join(config.DBPATH_PREFIX, "test_archival.txt")
self.logger = logger
def archive_files(self, files: List[str], archive_name: str, display_name: str):
def archive_files(self, files: list[str], archive_name: str, display_name: str):
self.logger.info(f"These files/directories would have been archived: {files}")
with open(self.archive_file, "a") as f:
for file in files:
@ -367,7 +366,7 @@ class Archival(object):
# Lock to control access from multiple threads.
self._lock = threading.Lock()
def archive_files(self, input_files: str | List[str], archive_name: str, display_name: str):
def archive_files(self, input_files: str | list[str], archive_name: str, display_name: str):
if isinstance(input_files, str):
input_files = [input_files]

12
buildscripts/resmokelib/utils/evergreen_conn.py
View File

@ -4,7 +4,7 @@ import os
import pathlib
from collections import deque
from pathlib import Path
from typing import Deque, Iterator, List, Optional, Set, Union
from typing import Iterator, Optional, Union
import requests
import structlog
@ -34,9 +34,9 @@ class EvergreenConnError(Exception):
pass
def _find_evergreen_yaml_candidates() -> List[str]:
def _find_evergreen_yaml_candidates() -> list[str]:
# Common for machines in Evergreen
candidates: List[Union[str, Path]] = [os.getcwd()]
candidates: list[Union[str, Path]] = [os.getcwd()]
cwd = pathlib.Path(os.getcwd())
# add every path that is the parent of CWD as well
@ -199,7 +199,7 @@ def get_compile_artifact_urls(
evg_build = evg_api.build_by_id(build_id)
LOGGER.debug("Found evergreen build.", evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}")
evg_tasks: Deque[Union[Task, str]] = deque(evg_build.get_tasks())
evg_tasks: deque[Union[Task, str]] = deque(evg_build.get_tasks())
tasks_wrapper = _filter_successful_tasks(evg_api, evg_tasks)
LOGGER.info(
"Found the following multiversion tasks",
@ -273,7 +273,7 @@ def _get_multiversion_urls(tasks_wrapper: _MultiversionTasks):
def _filter_successful_tasks(
evg_api: RetryingEvergreenApi, evg_tasks: Deque[Union[Task, str]]
evg_api: RetryingEvergreenApi, evg_tasks: deque[Union[Task, str]]
) -> _MultiversionTasks:
"""
We want to filter successful tasks in order by variant then by dependent tasks to find the compile tasks.
@ -283,7 +283,7 @@ def _filter_successful_tasks(
compile_task = None
archive_symbols_task = None
push_task = None
seen_task_ids: Set[str] = set()
seen_task_ids: set[str] = set()
while evg_tasks:
evg_task = evg_tasks.popleft()

4
buildscripts/resmokelib/utils/otel_id_generator.py
View File

@ -1,6 +1,6 @@
import random
import time
from typing import List, Optional
from typing import Optional
from opentelemetry import trace
from opentelemetry.sdk.trace import IdGenerator
@ -19,7 +19,7 @@ class ResmokeOtelIdGenerator(IdGenerator):
run in parallel with the same traceID and parentSpanID.
"""
def __init__(self, suite_files: Optional[List[str]] = None, shard_index: Optional[int] = None):
def __init__(self, suite_files: Optional[list[str]] = None, shard_index: Optional[int] = None):
"""
Initialize the unique span ID generator.

3
buildscripts/sbom_linter.py
View File

@ -2,7 +2,6 @@ import argparse
import json
import os
import sys
from typing import List
import jsonschema
from license_expression import get_spdx_licensing
@ -43,7 +42,7 @@ class ErrorManager:
def __init__(self, input_file: str):
self.input_file: str = input_file
self.component_name: str = ""
self.errors: List[str] = []
self.errors: list[str] = []
def update_component_attribute(self, component_name: str) -> None:
self.component_name = component_name

5
buildscripts/simple_report.py
View File

@ -3,7 +3,6 @@
import json
import os
import pathlib
from typing import List
import click
from typing_extensions import TypedDict
@ -25,7 +24,7 @@ class Report(TypedDict):
"""Evergreen report."""
failures: int
results: List[Result]
results: list[Result]
def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str:
@ -68,7 +67,7 @@ def try_combine_reports(out: Report):
pass
def _dedup_lines(lines: List[str]) -> List[str]:
def _dedup_lines(lines: list[str]) -> list[str]:
return list(set(lines))

32
buildscripts/smoke_tests/smoke_tests.py
View File

@ -24,7 +24,7 @@ from collections import defaultdict, deque
from dataclasses import dataclass
from pathlib import Path
from socket import gethostname
from typing import Any, Deque, Dict, List, Optional, Set, Union
from typing import Any, Optional, Union
from rich.status import Status
@ -62,10 +62,10 @@ from buildscripts.resmokelib.utils.evergreen_conn import get_evergreen_api
@dataclass
class Node:
name: str
args: List[str]
popen_kwargs: Dict[str, str]
args: list[str]
popen_kwargs: dict[str, str]
log_file: Path
deps: Set["Node"]
deps: set["Node"]
_start_time: Optional[float] = None
_finish_time: Optional[float] = None
_proc: Optional[subprocess.Popen] = None
@ -104,11 +104,11 @@ class Node:
self._finish_time = time.monotonic()
return self._proc.returncode
def deps_are_satisfied(self, finished: Set["Node"]):
def deps_are_satisfied(self, finished: set["Node"]):
return self.deps.issubset(finished)
def normalize_deps(x: Union[None, Node, Set[Node]]):
def normalize_deps(x: Union[None, Node, set[Node]]):
if x is None:
return set()
elif isinstance(x, (tuple, list, set)):
@ -117,7 +117,7 @@ def normalize_deps(x: Union[None, Node, Set[Node]]):
return {x}
def send_slack_notification(nodes: List[Node], total_elapsed: float, component_name: str):
def send_slack_notification(nodes: list[Node], total_elapsed: float, component_name: str):
overall_success = True
lines = [
"```",
@ -176,11 +176,11 @@ class CommandRunner:
):
self._log_path = log_path
self._parallelism = parallelism
self._downstream: Dict[Node, Set[Node]] = defaultdict(set)
self._nodes: Set[Node] = set()
self._finished: Set[Node] = set()
self._ready: Deque[Node] = deque()
self._running: Set[Node] = set()
self._downstream: dict[Node, set[Node]] = defaultdict(set)
self._nodes: set[Node] = set()
self._finished: set[Node] = set()
self._ready: deque[Node] = deque()
self._running: set[Node] = set()
self._status = Status(status=f"{component_name} smoke tests")
self._start_time = time.monotonic()
self._finish_time: Optional[float] = None
@ -205,9 +205,9 @@ class CommandRunner:
self,
*,
name: str,
args: List[Any],
args: list[Any],
log_file: str,
deps: Union[None, Node, Set[Node]] = None,
deps: Union[None, Node, set[Node]] = None,
**kwargs,
) -> Node:
log_file = self._log_path.joinpath(log_file)
@ -234,7 +234,7 @@ class CommandRunner:
print(f"Logging results to {self._log_path}")
self._status.start()
try:
iter_finished: Set[Node] = set()
iter_finished: set[Node] = set()
while self._finished != self._nodes:
while len(self._running) < self._parallelism and len(self._ready) > 0:
node = self._ready.popleft()
@ -327,7 +327,7 @@ def run_smoke_tests(
*,
component_name,
log_path: Path,
bazel_args: List[str],
bazel_args: list[str],
run_clang_tidy: bool,
send_slack_notification: bool,
):

3
buildscripts/tests/resmoke_end2end/test_resmoke.py
View File

@ -12,7 +12,6 @@ import sys
import time
import unittest
from shutil import rmtree
from typing import List
import yaml
@ -674,7 +673,7 @@ class TestDiscovery(_ResmokeSelftest):
)
def execute_resmoke(resmoke_args: List[str], subcommand: str = "run"):
def execute_resmoke(resmoke_args: list[str], subcommand: str = "run"):
return subprocess.run(
[sys.executable, "buildscripts/resmoke.py", subcommand] + resmoke_args,
text=True,

45
buildscripts/tests/resmokelib/testing/hooks/test_generate_and_check_perf_results.py
View File

@ -4,7 +4,6 @@
import datetime
import logging
import unittest
from typing import Dict, List
import mock
@ -373,7 +372,7 @@ class TestBenchmarkThreadsReport(GenerateAndCheckPerfResultsFixture):
class TestCheckPerfResultTestCase(unittest.TestCase):
def test_all_metrics_pass(self):
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -383,7 +382,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=1)
@ -402,7 +401,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
test_case.run_test()
def test_a_metric_fails(self):
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -412,7 +411,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -457,7 +456,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
mock_repo.get_pull.return_value = mock_pr
mock_github.get_repo.return_value = mock_repo
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -467,7 +466,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -513,7 +512,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
mock_repo.get_pull.return_value = mock_pr
mock_github.get_repo.return_value = mock_repo
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -523,7 +522,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -569,7 +568,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
mock_repo.get_pull.return_value = mock_pr
mock_github.get_repo.return_value = mock_repo
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -579,7 +578,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -625,7 +624,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
mock_repo.get_pull.return_value = mock_pr
mock_github.get_repo.return_value = mock_repo
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -635,7 +634,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -682,7 +681,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
mock_repo.get_pull.return_value = mock_pr
mock_github.get_repo.return_value = mock_repo
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -692,7 +691,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -723,7 +722,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
"github_token_mongo": "fake_token",
}.get(key, default)
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -733,7 +732,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -763,7 +762,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
"github_token_mongo": "fake_token",
}.get(key, default)
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -773,7 +772,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=100)
@ -793,7 +792,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
test_case.run_test()
def test_metric_doesnt_exist(self):
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -803,7 +802,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=1, metric_name="instructions"
): CedarMetric(name="instructions", type="LATENCY", value=1)
@ -822,7 +821,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
test_case.run_test()
def test_thread_level_doesnt_exist(self):
thresholds_to_check: List[cbr.IndividualMetricThreshold] = [
thresholds_to_check: list[cbr.IndividualMetricThreshold] = [
cbr.IndividualMetricThreshold(
metric_name="latency",
thread_level=1,
@ -832,7 +831,7 @@ class TestCheckPerfResultTestCase(unittest.TestCase):
threshold_limit=20,
)
]
reported_metrics: Dict[cbr.ReportedMetric, CedarMetric] = {
reported_metrics: dict[cbr.ReportedMetric, CedarMetric] = {
cbr.ReportedMetric(
test_name="fake-test", thread_level=12, metric_name="latency"
): CedarMetric(name="latency", type="LATENCY", value=1)

4
buildscripts/timeouts/timeout_service.py
View File

@ -1,6 +1,6 @@
"""Service for determining task timeouts."""
from typing import Any, Dict, NamedTuple, Optional
from typing import Any, NamedTuple, Optional
import inject
import structlog
@ -209,7 +209,7 @@ class TimeoutService:
return clean_every_n_cadence
def _get_hook_config(self, suite_name: str, hook_name: str) -> Optional[Dict[str, Any]]:
def _get_hook_config(self, suite_name: str, hook_name: str) -> Optional[dict[str, Any]]:
"""
Get the configuration for the given hook.

10
buildscripts/todo_check.py
View File

@ -6,7 +6,7 @@ import re
import sys
from collections import defaultdict
from dataclasses import dataclass
from typing import Callable, Dict, Iterable, List, NamedTuple, Optional
from typing import Callable, Iterable, NamedTuple, Optional
import click
@ -70,9 +70,9 @@ class FoundTodos:
by_file: All the references found mapped by the files they were found in.
"""
no_tickets: List[Todo]
with_tickets: Dict[str, List[Todo]]
by_file: Dict[str, List[Todo]]
no_tickets: list[Todo]
with_tickets: dict[str, list[Todo]]
by_file: dict[str, list[Todo]]
class TodoChecker:
@ -111,7 +111,7 @@ class TodoChecker:
walk_fs(base_dir, self.check_file)
@staticmethod
def print_todo_list(todo_list: List[Todo]) -> None:
def print_todo_list(todo_list: list[Todo]) -> None:
"""Display all the TODOs in the given list."""
last_file = None
for todo in todo_list:

5
buildscripts/tracing_profiler/profilerlib.py
View File

@ -1,6 +1,5 @@
import json
from dataclasses import dataclass
from typing import Dict
@dataclass
@ -12,7 +11,7 @@ class SpanMetrics:
net_nanos: int
exclusive_nanos: int
count: int
children: Dict[str, int]
children: dict[str, int]
def to_dict(self):
return {
@ -28,7 +27,7 @@ class SpanMetrics:
@dataclass
class CallMetrics:
spans: Dict[int, SpanMetrics]
spans: dict[int, SpanMetrics]
@staticmethod
def new_empty():

21
buildscripts/util/buildozer_utils.py
View File

@ -1,8 +1,7 @@
import subprocess
from typing import List
def _bd_command(cmd: str, labels: List[str]):
def _bd_command(cmd: str, labels: list[str]):
print(
f"buildozer '{cmd}' " + " ".join(labels),
)
@ -19,15 +18,15 @@ def _bd_command(cmd: str, labels: List[str]):
return p
def bd_add(labels: List[str], attr: str, values: List[str]) -> None:
def bd_add(labels: list[str], attr: str, values: list[str]) -> None:
_bd_command(f'add {attr} {" ".join(values)}', labels)
def bd_set(labels: List[str], attr: str, value: str) -> None:
def bd_set(labels: list[str], attr: str, value: str) -> None:
_bd_command(f"set {attr} {value}", labels)
def bd_remove(labels: List[str], attr: str, values: List[str]) -> None:
def bd_remove(labels: list[str], attr: str, values: list[str]) -> None:
_bd_command(f'remove {attr} {" ".join(values)}', labels)
@ -35,26 +34,26 @@ def bd_new(package: str, rule_kind: str, rule_name: str) -> None:
_bd_command(f"new {rule_kind} {rule_name}", [package])
def bd_comment(labels: List[str], comment: str, attr: str = "", value: str = "") -> None:
def bd_comment(labels: list[str], comment: str, attr: str = "", value: str = "") -> None:
_bd_command(f"comment {attr} {value} {comment}", labels)
def bd_print(labels: List[str], attrs: List[str]) -> str:
def bd_print(labels: list[str], attrs: list[str]) -> str:
p = _bd_command(f'print {" ".join(attrs)}', labels)
return p.stdout
def bd_new_load(packages: List[str], path: str, rules: List[str]) -> None:
def bd_new_load(packages: list[str], path: str, rules: list[str]) -> None:
_bd_command(f'new_load {path} {" ".join(rules)}', packages)
def bd_fix(fixes: List[str]) -> None:
def bd_fix(fixes: list[str]) -> None:
_bd_command(f'fix {" ".join(fixes)}', [])
def bd_copy(labels: List[str], attr: str, from_rule: str) -> None:
def bd_copy(labels: list[str], attr: str, from_rule: str) -> None:
_bd_command(f"copy {attr} {from_rule}", labels)
def bd_move(labels: List[str], old_attr: str, new_attr: str) -> None:
def bd_move(labels: list[str], old_attr: str, new_attr: str) -> None:
_bd_command(f"move {old_attr} {new_attr} *", labels)

4
buildscripts/util/cedar_report.py
View File

@ -1,7 +1,7 @@
"""Cedar report."""
from dataclasses import dataclass
from typing import List, Union
from typing import Union
@dataclass
@ -29,7 +29,7 @@ class CedarTestReport:
test_name: str
thread_level: int
metrics: List[CedarMetric]
metrics: list[CedarMetric]
def as_dict(self) -> dict:
"""Return dictionary representation."""

9
buildscripts/util/codeowners_utils.py
View File

@ -2,13 +2,12 @@ import fnmatch
import os
import re
from functools import lru_cache
from typing import Dict, List, Tuple
import yaml
@lru_cache
def process_owners(cur_dir: str) -> Tuple[Dict[re.Pattern, List[str]], bool]:
def process_owners(cur_dir: str) -> tuple[dict[re.Pattern, list[str]], bool]:
if not cur_dir:
return ({}, False)
@ -54,7 +53,7 @@ class Owners:
open("buildscripts/util/co_jira_map.yml", "r", encoding="utf8")
)
def get_codeowners(self, file_path: str) -> List[str]:
def get_codeowners(self, file_path: str) -> list[str]:
cur_dir = os.path.dirname(file_path)
codeowners = []
# search up tree until matching filter found
@ -71,10 +70,10 @@ class Owners:
cur_dir = os.path.dirname(cur_dir)
return codeowners
def get_jira_team_from_codeowner(self, codeowner: str) -> List[str]:
def get_jira_team_from_codeowner(self, codeowner: str) -> list[str]:
return self.co_jira_map[codeowner]
def get_jira_team_owner(self, file_path: str) -> List[str]:
def get_jira_team_owner(self, file_path: str) -> list[str]:
return [
jira_team
for codeowner in self.get_codeowners(file_path)

Some files were not shown because too many files have changed in this diff Show More