SERVER-106697: Speed up reading main evergreen yaml (#37717)

GitOrigin-RevId: 45fa8f9d43ec165a584539f1739ed31adbc062e3

buildscripts/ciconfig/BUILD.bazel

@@ -5,6 +5,7 @@ py_library(
     srcs = [
         "__init__.py",
         "evergreen.py",
+        "yaml_load.py",
     ],
     visibility = ["//visibility:public"],
     deps = [

buildscripts/ciconfig/evergreen.py

@@ -15,7 +15,8 @@ import sys
 from typing import Any, Dict, List, Optional, Set
 
 import structlog
-import yaml
+
+from buildscripts.ciconfig.yaml_load import yaml_load
 
 ENTERPRISE_MODULE_NAME = "enterprise"
 ASAN_SIGNATURE = "detect_leaks=1"
@@ -80,10 +81,11 @@ def parse_evergreen_file(path, evergreen_binary="evergreen"):
                     path, result.stdout, result.stderr
                 )
             )
-        config = yaml.safe_load(result.stdout)
+        config: dict = yaml_load(result.stdout)
     else:
         with open(path, "r", encoding="utf8") as fstream:
-            config = yaml.safe_load(fstream)
+            data = fstream.read()
+            config: dict = yaml_load(data)
 
     return EvergreenProjectConfig(config)
 

buildscripts/ciconfig/yaml_load.py

@@ -0,0 +1,46 @@
+from typing import Any
+
+# PyYaml is very easy to use, but it is very slow. This is a problem for us since the main evergreen.yml file is quite large.
+# PyYaml was taking over 10s to just load the file, which needed to be done every single task and so was a significant bottleneck.
+# We use the rapidyaml library instead, which is much more low level but much faster (sub 1s to load the same file). This is not a
+# full drop in replacement for PyYaml and does not fully satisfy the yaml spec, but it is sufficient for our needs.
+
+
+try:
+    import ryml
+
+    def ryml_to_dict(tree: ryml.Tree, index: int = 0) -> Any:
+        """Walk through the ryml tree and convert nodes."""
+        if tree.is_map(index):
+            return {
+                str(tree.key(child_index), "utf8"): ryml_to_dict(tree, child_index)
+                for child_index in ryml.children(tree, index)
+            }
+        elif tree.is_seq(index):
+            return [ryml_to_dict(tree, child_index) for child_index in ryml.children(tree, index)]
+        else:
+            decoded_value = tree.val(index).tobytes().decode("utf8")
+            if decoded_value == "true":
+                return True
+            elif decoded_value == "false":
+                return False
+            elif decoded_value == "null" or decoded_value == "~":
+                return None
+            try:
+                int_value = int(decoded_value)
+                return int_value
+            except ValueError:
+                pass
+            try:
+                float_value = float(decoded_value)
+                return float_value
+            except ValueError:
+                pass
+            return decoded_value
+
+    def yaml_load(data: str) -> dict:
+        """Safely load YAML data."""
+        return ryml_to_dict(ryml.parse_in_arena(data))
+
+except ImportError:
+    from yaml import safe_load as yaml_load  # noqa

buildscripts/tests/resmoke_end2end/test_resmoke.py

@@ -808,7 +808,7 @@ class TestEvergreenYML(unittest.TestCase):
             generate_func = task.find_func_command("generate resmoke tasks")
             if (
                 generate_func is None
-                or get_dict_value(generate_func, ["vars", "is_jstestfuzz"]) != "true"
+                or get_dict_value(generate_func, ["vars", "is_jstestfuzz"]) is not True
             ):
                 continue
 

poetry.lock

@@ -695,6 +695,22 @@ wrapt = ">=1.10,<2"
 [package.extras]
 dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"]
 
+[[package]]
+name = "deprecation"
+version = "2.1.0"
+description = "A library to handle automated deprecations"
+optional = false
+python-versions = "*"
+groups = ["core"]
+markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\""
+files = [
+    {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"},
+    {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"},
+]
+
+[package.dependencies]
+packaging = "*"
+
 [[package]]
 name = "distlib"
 version = "0.3.9"
@@ -2459,7 +2475,7 @@ version = "24.2"
 description = "Core utilities for Python packages"
 optional = false
 python-versions = ">=3.8"
-groups = ["compile", "export", "idl", "testing"]
+groups = ["compile", "core", "export", "idl", "testing"]
 markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
 files = [
     {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
@@ -3497,6 +3513,26 @@ files = [
 [package.extras]
 all = ["numpy"]
 
+[[package]]
+name = "rapidyaml"
+version = "0.9.0.post2"
+description = "Rapid YAML - a library to parse and emit YAML, and do it fast"
+optional = false
+python-versions = ">=3.6"
+groups = ["core"]
+markers = "(platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\") and platform_system != \"Windows\""
+files = []
+develop = false
+
+[package.dependencies]
+deprecation = "*"
+
+[package.source]
+type = "git"
+url = "https://github.com/biojppm/rapidyaml"
+reference = "HEAD"
+resolved_reference = "338385b46a312bd8e05016033dd452dccc67e372"
+
 [[package]]
 name = "referencing"
 version = "0.36.2"
@@ -5411,4 +5447,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<4.0"
-content-hash = "4adbaf602d1abb198a2f0339c0882e862e61490616ee5002559e8a324bcf7548"
+content-hash = "0f66bc1087116663ea12e384ed3266d07d495738b0e57eaddcc0b6b029a4b66e"

pyproject.toml

@@ -73,6 +73,7 @@ requests = "^2.32.3"
 typing-extensions = "^4.12.2"
 typer = "^0.12.3"
 tenacity = "^9.0.0"
+rapidyaml = {git = "https://github.com/biojppm/rapidyaml@v0.9.0", markers = "platform_system != 'Windows'"}
 
 [tool.poetry.group.export.dependencies]
 pipx = "1.6.0"
@@ -282,4 +283,4 @@ reportConstantRedefinition = "none"
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"