From eb8d3fcbf2579e413d1173f445137dfd16cbc8d3 Mon Sep 17 00:00:00 2001 From: Jeff Zambory <37782453+zamj@users.noreply.github.com> Date: Wed, 25 Jun 2025 12:33:21 -0400 Subject: [PATCH] SERVER-106697: Speed up reading main evergreen yaml (#37717) GitOrigin-RevId: 45fa8f9d43ec165a584539f1739ed31adbc062e3 --- buildscripts/ciconfig/BUILD.bazel | 1 + buildscripts/ciconfig/evergreen.py | 8 ++-- buildscripts/ciconfig/yaml_load.py | 46 +++++++++++++++++++ .../tests/resmoke_end2end/test_resmoke.py | 2 +- poetry.lock | 40 +++++++++++++++- pyproject.toml | 3 +- 6 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 buildscripts/ciconfig/yaml_load.py diff --git a/buildscripts/ciconfig/BUILD.bazel b/buildscripts/ciconfig/BUILD.bazel index b80c6e14e2b..7488fb682e1 100644 --- a/buildscripts/ciconfig/BUILD.bazel +++ b/buildscripts/ciconfig/BUILD.bazel @@ -5,6 +5,7 @@ py_library( srcs = [ "__init__.py", "evergreen.py", + "yaml_load.py", ], visibility = ["//visibility:public"], deps = [ diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index b624671b9ae..47b1e48df23 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -15,7 +15,8 @@ import sys from typing import Any, Dict, List, Optional, Set import structlog -import yaml + +from buildscripts.ciconfig.yaml_load import yaml_load ENTERPRISE_MODULE_NAME = "enterprise" ASAN_SIGNATURE = "detect_leaks=1" @@ -80,10 +81,11 @@ def parse_evergreen_file(path, evergreen_binary="evergreen"): path, result.stdout, result.stderr ) ) - config = yaml.safe_load(result.stdout) + config: dict = yaml_load(result.stdout) else: with open(path, "r", encoding="utf8") as fstream: - config = yaml.safe_load(fstream) + data = fstream.read() + config: dict = yaml_load(data) return EvergreenProjectConfig(config) diff --git a/buildscripts/ciconfig/yaml_load.py b/buildscripts/ciconfig/yaml_load.py new file mode 100644 index 00000000000..505ae21af99 --- /dev/null +++ b/buildscripts/ciconfig/yaml_load.py @@ -0,0 +1,46 @@ +from typing import Any + +# PyYaml is very easy to use, but it is very slow. This is a problem for us since the main evergreen.yml file is quite large. +# PyYaml was taking over 10s to just load the file, which needed to be done every single task and so was a significant bottleneck. +# We use the rapidyaml library instead, which is much more low level but much faster (sub 1s to load the same file). This is not a +# full drop in replacement for PyYaml and does not fully satisfy the yaml spec, but it is sufficient for our needs. + + +try: + import ryml + + def ryml_to_dict(tree: ryml.Tree, index: int = 0) -> Any: + """Walk through the ryml tree and convert nodes.""" + if tree.is_map(index): + return { + str(tree.key(child_index), "utf8"): ryml_to_dict(tree, child_index) + for child_index in ryml.children(tree, index) + } + elif tree.is_seq(index): + return [ryml_to_dict(tree, child_index) for child_index in ryml.children(tree, index)] + else: + decoded_value = tree.val(index).tobytes().decode("utf8") + if decoded_value == "true": + return True + elif decoded_value == "false": + return False + elif decoded_value == "null" or decoded_value == "~": + return None + try: + int_value = int(decoded_value) + return int_value + except ValueError: + pass + try: + float_value = float(decoded_value) + return float_value + except ValueError: + pass + return decoded_value + + def yaml_load(data: str) -> dict: + """Safely load YAML data.""" + return ryml_to_dict(ryml.parse_in_arena(data)) + +except ImportError: + from yaml import safe_load as yaml_load # noqa diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 733a23db74a..ae2a5b96629 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -808,7 +808,7 @@ class TestEvergreenYML(unittest.TestCase): generate_func = task.find_func_command("generate resmoke tasks") if ( generate_func is None - or get_dict_value(generate_func, ["vars", "is_jstestfuzz"]) != "true" + or get_dict_value(generate_func, ["vars", "is_jstestfuzz"]) is not True ): continue diff --git a/poetry.lock b/poetry.lock index 63e333082c5..d1f36081a20 100644 --- a/poetry.lock +++ b/poetry.lock @@ -695,6 +695,22 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +[[package]] +name = "deprecation" +version = "2.1.0" +description = "A library to handle automated deprecations" +optional = false +python-versions = "*" +groups = ["core"] +markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\"" +files = [ + {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, + {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, +] + +[package.dependencies] +packaging = "*" + [[package]] name = "distlib" version = "0.3.9" @@ -2459,7 +2475,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["compile", "export", "idl", "testing"] +groups = ["compile", "core", "export", "idl", "testing"] markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\"" files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, @@ -3497,6 +3513,26 @@ files = [ [package.extras] all = ["numpy"] +[[package]] +name = "rapidyaml" +version = "0.9.0.post2" +description = "Rapid YAML - a library to parse and emit YAML, and do it fast" +optional = false +python-versions = ">=3.6" +groups = ["core"] +markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\"" +files = [] +develop = false + +[package.dependencies] +deprecation = "*" + +[package.source] +type = "git" +url = "https://github.com/biojppm/rapidyaml" +reference = "HEAD" +resolved_reference = "338385b46a312bd8e05016033dd452dccc67e372" + [[package]] name = "referencing" version = "0.36.2" @@ -5411,4 +5447,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "4adbaf602d1abb198a2f0339c0882e862e61490616ee5002559e8a324bcf7548" +content-hash = "0f66bc1087116663ea12e384ed3266d07d495738b0e57eaddcc0b6b029a4b66e" diff --git a/pyproject.toml b/pyproject.toml index 73bdfcc405f..d85ddb9f699 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ requests = "^2.32.3" typing-extensions = "^4.12.2" typer = "^0.12.3" tenacity = "^9.0.0" +rapidyaml = {git = "https://github.com/biojppm/rapidyaml@v0.9.0", markers = "platform_system != 'Windows'"} [tool.poetry.group.export.dependencies] pipx = "1.6.0" @@ -282,4 +283,4 @@ reportConstantRedefinition = "none" [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +build-backend = "poetry.core.masonry.api" \ No newline at end of file