diff --git a/buildscripts/idl/BUILD.bazel b/buildscripts/idl/BUILD.bazel index 96d61768706..a1af5f4433c 100644 --- a/buildscripts/idl/BUILD.bazel +++ b/buildscripts/idl/BUILD.bazel @@ -14,6 +14,7 @@ py_library( name = "idl", srcs = [ "gen_all_feature_flag_list.py", + "generate_ifr_registry.py", "idlc.py", "lib.py", ] + glob(["idl/**/*.py"]), diff --git a/buildscripts/idl/generate_ifr_registry.py b/buildscripts/idl/generate_ifr_registry.py new file mode 100755 index 00000000000..9f97db1f01d --- /dev/null +++ b/buildscripts/idl/generate_ifr_registry.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Generate src/mongo/db/ifr_flag_registry.yml from the feature flags in IDL files. + +The IFR flag registry lists every feature flag declared with +incremental_rollout_phase 'rollout' or 'released' across all IDL files under src/. +It is fully derived from the IDL. + +The registry's purpose is to give GitHub's CODEOWNERS mechanism a concrete +path to gate: OWNERS.yml lists the registry under @10gen/rollout-flag-managers, +so every regenerated change triggers that team's review when committed. See the +generated file's header for the full rationale. +""" + +import argparse +import sys +import textwrap +from pathlib import Path + +import yaml + +DEFAULT_REGISTRY_PATH = "src/mongo/db/ifr_flag_registry.yml" +DEFAULT_SOURCE_ROOT = "src" + +_HEADER = textwrap.dedent("""\ + # AUTO-GENERATED — do not edit by hand. + # + # This file is regenerated by buildscripts/idl/generate_ifr_registry.py. + # It lists every feature flag declared with incremental_rollout_phase + # 'rollout' or 'released' across the IDL sources under src/. Just commit + # whatever your script invocation produces — there is no manual step. + # + # Why this file exists + # -------------------- + # The registry is a hook for GitHub's CODEOWNERS mechanism. OWNERS.yml + # lists this file under @10gen/rollout-flag-managers, so every regenerated + # change requires that team's review — which is how the team stays informed + # about each phase transition. + # + # This ownership and heavy review process is expected to be a temporary + # measure while the organization continues to improve the IFR tooling, and + # while the broader server org ramps up its familiarity with IFR. The + # long-term goal is to have the phase transition process be as lightweight + # as possible for server developers, while still giving rollout flag + # managers confidence that they won't miss any important changes. This file + # and its associated CODEOWNERS gate are part of the current solution to + # that problem, but may not be needed in the future as the ecosystem + # matures. + # + # What IFR is + # ----------- + # Incremental Feature Rollout is MongoDB's mechanism for shipping a feature + # as on-by-default while still allowing it to be toggled at runtime, so + # operators can disable it without a server restart if something goes wrong. + # - 'rollout': actively being rolled out; on by default, kill-switchable. + # - 'released': rollout complete; retained here as a historical record. +""") + + +def _iter_idl_files(source_root): + """Yield absolute paths for every .idl file under source_root, sorted.""" + root = Path(source_root) + if not root.is_dir(): + return + yield from sorted(root.rglob("*.idl")) + + +def _extract_phase_flags(idl_path): + """Return (rollout_flags, released_flags) declared in one IDL file. + + IDL files are YAML, so we can read them directly with yaml.safe_load — no need + to bring in the full IDL parser just to look at feature_flags entries. + """ + with open(idl_path, encoding="utf-8") as f: + doc = yaml.safe_load(f) + if not isinstance(doc, dict): + return set(), set() + feature_flags = doc.get("feature_flags") or {} + rollout = set() + released = set() + for flag_name, flag_body in feature_flags.items(): + if not isinstance(flag_body, dict): + continue + phase = flag_body.get("incremental_rollout_phase") + if phase == "rollout": + rollout.add(flag_name) + elif phase == "released": + released.add(flag_name) + return rollout, released + + +def collect_flags(source_root): + """Walk source_root and return (rollout_flags, released_flags) across all IDLs.""" + rollout = set() + released = set() + for idl_path in _iter_idl_files(source_root): + r, rel = _extract_phase_flags(idl_path) + rollout |= r + released |= rel + overlap = rollout & released + if overlap: + # TODO SERVER-126893 This shouldn't be possible, and we should delete + # this code and the corresponding test case. + raise ValueError( + f"Feature flag(s) declared in both 'rollout' and 'released' phases " + f"across IDL files: {sorted(overlap)}. A flag can only be in one phase." + ) + return rollout, released + + +def render_registry(rollout_flags, released_flags): + """Render the registry file contents. Deterministic given the inputs.""" + + def section(name, flags): + if not flags: + return f"{name}: {{}}\n" + lines = [f"{name}:"] + # Empty-mapping values keep the shape stable so future metadata + # (e.g. since_version) can be added without a format change. + lines.extend(f" {flag}: {{}}" for flag in sorted(flags)) + return "\n".join(lines) + "\n" + + return ( + _HEADER + + "\n" + + section("rollout", rollout_flags) + + "\n" + + section("released", released_flags) + ) + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--source-root", + default=DEFAULT_SOURCE_ROOT, + help="Source tree to scan for .idl files (default: %(default)s).", + ) + parser.add_argument( + "--output", + default=DEFAULT_REGISTRY_PATH, + help="Path to the registry file (default: %(default)s).", + ) + args = parser.parse_args(argv) + + try: + rollout, released = collect_flags(args.source_root) + except ValueError as exc: + sys.stderr.write(f"{exc}\n") + return 2 + + with open(args.output, "w", encoding="utf-8") as f: + f.write(render_registry(rollout, released)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/buildscripts/idl/tests/test_generate_ifr_registry.py b/buildscripts/idl/tests/test_generate_ifr_registry.py new file mode 100644 index 00000000000..1d5acdac64e --- /dev/null +++ b/buildscripts/idl/tests/test_generate_ifr_registry.py @@ -0,0 +1,251 @@ +# Copyright (C) 2026-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the Server Side Public License, version 1, as published by +# MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Server +# Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# . +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you do not wish +# to do so, delete this exception statement from your version. If you +# delete this exception statement from all source files in the program, +# then also delete it in the license file. +# +"""Test cases for buildscripts/idl/generate_ifr_registry.py.""" + +import os +import sys +import tempfile +import textwrap +import unittest + +# Allow running via pytest from the repo root. +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import generate_ifr_registry # noqa: E402 + + +class TestGenerateIFRRegistry(unittest.TestCase): + """Test generate_ifr_registry against synthetic and real IDL trees.""" + + def _write(self, path, content): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(textwrap.dedent(content)) + + def test_collect_flags_picks_up_rollout_and_released(self): + with tempfile.TemporaryDirectory() as root: + self._write( + os.path.join(root, "a.idl"), + """ + feature_flags: + featureFlagAlpha: + incremental_rollout_phase: rollout + featureFlagBeta: + incremental_rollout_phase: in_development + """, + ) + self._write( + os.path.join(root, "sub", "b.idl"), + """ + feature_flags: + featureFlagGamma: + incremental_rollout_phase: released + """, + ) + rollout, released = generate_ifr_registry.collect_flags(root) + self.assertEqual(rollout, {"featureFlagAlpha"}) + self.assertEqual(released, {"featureFlagGamma"}) + + def test_collect_flags_raises_on_overlap(self): + # TODO SERVER-126893 delete this test. + with tempfile.TemporaryDirectory() as root: + self._write( + os.path.join(root, "one.idl"), + """ + feature_flags: + featureFlagDup: + incremental_rollout_phase: rollout + """, + ) + self._write( + os.path.join(root, "two.idl"), + """ + feature_flags: + featureFlagDup: + incremental_rollout_phase: released + """, + ) + with self.assertRaisesRegex(ValueError, "featureFlagDup"): + generate_ifr_registry.collect_flags(root) + + def test_collect_flags_ignores_non_idl_yaml_and_missing_sections(self): + with tempfile.TemporaryDirectory() as root: + # An IDL file without a feature_flags section. + self._write( + os.path.join(root, "noflags.idl"), + """ + global: + cpp_namespace: mongo + """, + ) + # A sibling YAML file that isn't an .idl — should be skipped by the glob. + self._write( + os.path.join(root, "other.yml"), + """ + feature_flags: + featureFlagShouldNotAppear: + incremental_rollout_phase: rollout + """, + ) + rollout, released = generate_ifr_registry.collect_flags(root) + self.assertEqual(rollout, set()) + self.assertEqual(released, set()) + + def test_render_produces_stable_sorted_output(self): + body = generate_ifr_registry.render_registry( + {"featureFlagB", "featureFlagA"}, {"featureFlagC"} + ) + # Flags appear in sorted order, regardless of input set ordering. + rollout_idx_a = body.index("featureFlagA") + rollout_idx_b = body.index("featureFlagB") + self.assertLess(rollout_idx_a, rollout_idx_b) + self.assertIn("released:\n featureFlagC: {}", body) + + def test_render_empty_sections_use_flow_style_empty_map(self): + body = generate_ifr_registry.render_registry(set(), set()) + self.assertIn("rollout: {}", body) + self.assertIn("released: {}", body) + + def test_collect_flags_ignores_unrelated_idl_content(self): + """Adding non-flag content to an IDL file does not change the registry output.""" + with tempfile.TemporaryDirectory() as root: + self._write( + os.path.join(root, "a.idl"), + """ + feature_flags: + featureFlagAlpha: + incremental_rollout_phase: rollout + featureFlagBeta: + incremental_rollout_phase: released + """, + ) + rollout_before, released_before = generate_ifr_registry.collect_flags(root) + + # Add unrelated content before, after, and between the flags. + self._write( + os.path.join(root, "a.idl"), + """ + global: + cpp_namespace: mongo + server_parameters: + someParam: + set_at: startup + feature_flags: + featureFlagAlpha: + incremental_rollout_phase: rollout + featureFlagUnrelated: + incremental_rollout_phase: in_development + featureFlagBeta: + incremental_rollout_phase: released + commands: + someCommand: + description: "a command" + """, + ) + rollout_after, released_after = generate_ifr_registry.collect_flags(root) + + self.assertEqual(rollout_before, rollout_after) + self.assertEqual(released_before, released_after) + + def test_collect_flags_picks_up_new_flag_added_to_existing_idl(self): + """Adding a new rollout flag to an IDL that already has flags registers it.""" + with tempfile.TemporaryDirectory() as root: + self._write( + os.path.join(root, "a.idl"), + """ + feature_flags: + featureFlagAlpha: + incremental_rollout_phase: rollout + """, + ) + rollout_before, _ = generate_ifr_registry.collect_flags(root) + self.assertEqual(rollout_before, {"featureFlagAlpha"}) + + # Add a second rollout flag to the same file. + self._write( + os.path.join(root, "a.idl"), + """ + feature_flags: + featureFlagAlpha: + incremental_rollout_phase: rollout + featureFlagBeta: + incremental_rollout_phase: rollout + """, + ) + rollout_after, _ = generate_ifr_registry.collect_flags(root) + + self.assertEqual(rollout_after, {"featureFlagAlpha", "featureFlagBeta"}) + + +class TestRegistryStaleness(unittest.TestCase): + """Ensure the committed ifr_flag_registry.yml matches what the IDL sources produce.""" + + # Resolve repo root: this file lives at buildscripts/idl/tests/ + _REPO_ROOT = os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") + ) + + def test_committed_registry_is_up_to_date(self): + source_root = os.path.join(self._REPO_ROOT, generate_ifr_registry.DEFAULT_SOURCE_ROOT) + registry_path = os.path.join(self._REPO_ROOT, generate_ifr_registry.DEFAULT_REGISTRY_PATH) + + rollout, released = generate_ifr_registry.collect_flags(source_root) + expected = generate_ifr_registry.render_registry(rollout, released) + + self.assertTrue( + os.path.exists(registry_path), + f"{generate_ifr_registry.DEFAULT_REGISTRY_PATH} does not exist. " + f"Run: python3 buildscripts/idl/generate_ifr_registry.py", + ) + + with open(registry_path, encoding="utf-8") as f: + actual = f.read() + + if actual != expected: + # Build a human-friendly summary of what changed. + actual_lines = set(actual.splitlines()) + expected_lines = set(expected.splitlines()) + missing = sorted(expected_lines - actual_lines) + extra = sorted(actual_lines - expected_lines) + details = [] + if missing: + details.append("Lines missing from committed file:\n " + "\n ".join(missing)) + if extra: + details.append("Extra lines in committed file:\n " + "\n ".join(extra)) + diff_summary = "\n".join(details) if details else "(content differs)" + + self.fail( + f"{generate_ifr_registry.DEFAULT_REGISTRY_PATH} is out of date.\n" + f"\n" + f"{diff_summary}\n" + f"\n" + f"To fix, run:\n" + f" python3 buildscripts/idl/generate_ifr_registry.py\n" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/mongo/db/OWNERS.yml b/src/mongo/db/OWNERS.yml index 37f2ccf57d0..c384a9ca4ff 100644 --- a/src/mongo/db/OWNERS.yml +++ b/src/mongo/db/OWNERS.yml @@ -65,6 +65,9 @@ filters: - "generic_argument_util.*": approvers: - 10gen/server-programmability + - "ifr_flag_registry.yml": + approvers: + - 10gen/rollout-flag-managers - "ifr_flag_retry_info*": approvers: - 10gen/server-programmability diff --git a/src/mongo/db/ifr_flag_registry.yml b/src/mongo/db/ifr_flag_registry.yml new file mode 100644 index 00000000000..5d6a9b94742 --- /dev/null +++ b/src/mongo/db/ifr_flag_registry.yml @@ -0,0 +1,38 @@ +# AUTO-GENERATED — do not edit by hand. +# +# This file is regenerated by buildscripts/idl/generate_ifr_registry.py. +# It lists every feature flag declared with incremental_rollout_phase +# 'rollout' or 'released' across the IDL sources under src/. Just commit +# whatever your script invocation produces — there is no manual step. +# +# Why this file exists +# -------------------- +# The registry is a hook for GitHub's CODEOWNERS mechanism. OWNERS.yml +# lists this file under @10gen/rollout-flag-managers, so every regenerated +# change requires that team's review — which is how the team stays informed +# about each phase transition. +# +# This ownership and heavy review process is expected to be a temporary +# measure while the organization continues to improve the IFR tooling, and +# while the broader server org ramps up its familiarity with IFR. The +# long-term goal is to have the phase transition process be as lightweight +# as possible for server developers, while still giving rollout flag +# managers confidence that they won't miss any important changes. This file +# and its associated CODEOWNERS gate are part of the current solution to +# that problem, but may not be needed in the future as the ecosystem +# matures. +# +# What IFR is +# ----------- +# Incremental Feature Rollout is MongoDB's mechanism for shipping a feature +# as on-by-default while still allowing it to be toggled at runtime, so +# operators can disable it without a server restart if something goes wrong. +# - 'rollout': actively being rolled out; on by default, kill-switchable. +# - 'released': rollout complete; retained here as a historical record. + +rollout: + featureFlagCostBasedRanker: {} + featureFlagMultiPlanLimiter: {} + featureFlagUnifiedWriteExecutor: {} + +released: {}