mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-116221: Reference python3.13 in v5 toolchain (#47179)

Browse Source 
GitOrigin-RevId: d810136e7a469c63fa0051464b29f88924512d22
This commit is contained in:
Nick Jefferies 2026-02-05 11:02:51 -05:00 committed by MongoDB Bot
parent beb3aab2dd
commit 9f3133e98b
22 changed files with 231 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.devcontainer/Dockerfile
View File

@ -130,10 +130,10 @@ RUN echo 'export PATH="$HOME/.local/bin:${PATH}"' > /etc/profile.d/03-local-bin.
USER $USERNAME
ENV PATH="/home/${USERNAME}/.local/bin:${PATH}"
RUN /opt/mongodbtoolchain/v5/bin/python3 -m venv /tmp/pipx-venv && \
/tmp/pipx-venv/bin/python -m pip install --upgrade "pip<20.3" && \
RUN /opt/mongodbtoolchain/v5/bin/python3.13 -m venv /tmp/pipx-venv && \
/tmp/pipx-venv/bin/python -m pip install --upgrade "pip==25.3" && \
/tmp/pipx-venv/bin/python -m pip install pipx && \
/tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3 --force && \
/tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force && \
rm -rf /tmp/pipx-venv
# Note: PATH is configured via /etc/profile.d, not ~/.bashrc, to avoid modifying home volume

2
.devcontainer/post-create.sh
View File

@ -138,7 +138,7 @@ echo "Creating Python virtual environment..."
venv_created=false
if [ ! -d "${WORKSPACE_FOLDER}/python3-venv/bin" ]; then
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
/opt/mongodbtoolchain/v5/bin/python3 -m venv "${WORKSPACE_FOLDER}/python3-venv"
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv "${WORKSPACE_FOLDER}/python3-venv"
venv_created=true
# Install dependencies in the newly created venv

28
.devcontainer/toolchain_config.env
View File

@ -1,20 +1,20 @@
# Generated by toolchain.py
# DO NOT EDIT MANUALLY - run: python3 toolchain.py generate
# DO NOT EDIT MANUALLY - run: python3 toolchain.py
#
# Generated: 2025-10-01T12:25:08.235721
# Generated: 2026-01-30T19:44:10.636425
# ARM64 Toolchain
# Last Modified: 2025-07-21T20:49:55+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce
TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_ARM64_SHA256="cdd2a58ed4a67dfa1be237d6042b7523552b543bb104c20db8f29068f3899fd6"
TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_ARM64_LAST_MODIFIED="2025-07-21T20:49:55+00:00"
# Last Modified: 2026-01-30T18:35:20+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37
TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz"
TOOLCHAIN_ARM64_SHA256="d4baf5bff8f3ef053b234028a0d3148193eaa59840c8c0fc3544809cf2d4c9df"
TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz"
TOOLCHAIN_ARM64_LAST_MODIFIED="2026-01-30T18:35:20+00:00"
# AMD64 Toolchain
# Last Modified: 2025-07-21T19:57:00+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce
TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_AMD64_SHA256="55b927124ffbf040b0caf19cb7b440679d71e57ae29c1c40df0891b884dcd2cd"
TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_AMD64_LAST_MODIFIED="2025-07-21T19:57:00+00:00"
# Last Modified: 2026-01-16T22:50:49+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd
TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz"
TOOLCHAIN_AMD64_SHA256="aebfc49a6f7ee9c1430807086b964c3756d491048d68572c808af2fc2e805374"
TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz"
TOOLCHAIN_AMD64_LAST_MODIFIED="2026-01-16T22:50:49+00:00"

18
buildscripts/auto_install_db_contrib_tool.sh
View File

@ -25,8 +25,8 @@ if [[ -f "$HOME/.zshrc" ]]; then
fi
if ! command -v db-contrib-tool &>/dev/null; then
if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3 -c 'import sys; print(sys.version)')
if ! python3.13 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3.13 -c 'import sys; print(sys.version)')
echo "You must have python3.7+ installed. Detected version $actual_version."
echo "To avoid unexpected issues, python3.7+ will not be automatically installed."
echo "Please, do it yourself."
@ -48,28 +48,28 @@ if ! command -v db-contrib-tool &>/dev/null; then
source "$rc_file"
fi
pipx install db-contrib-tool --python $(command -v python3) --force
pipx install db-contrib-tool --python $(command -v python3.13) --force
echo
else
if ! python3 -m pipx --version &>/dev/null; then
if ! python3.13 -m pipx --version &>/dev/null; then
echo "Couldn't find pipx. Installing it as python3 module:"
echo " $(command -v python3) -m pip install pipx"
echo " $(command -v python3.13) -m pip install pipx"
echo
python3 -m pip install pipx
python3.13 -m pip install pipx
echo
else
echo "Found pipx installed as python3 module:"
echo " $(command -v python3) -m pipx --version"
echo " $(command -v python3.13) -m pipx --version"
echo "Using it to install 'db-contrib-tool'."
echo
fi
python3 -m pipx ensurepath &>/dev/null
python3.13 -m pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then
source "$rc_file"
fi
python3 -m pipx install db-contrib-tool --force
python3.13 -m pipx install db-contrib-tool --force
echo
fi
fi

2
buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py
View File

@ -175,7 +175,7 @@ config_fuzzer_params = {
# batch limit operations.
"replBatchLimitOperations": {
"min": 1,
"max": 0.2 * 1000 * 1000,
"max": int(0.2 * 1000 * 1000), # Must be int for randint() compatibility
"period": 5,
"fuzz_at": ["startup", "runtime"],
},

51
buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py
View File

@ -8,14 +8,26 @@ import stat
from buildscripts.resmokelib import config, utils
def generate_normal_wt_parameters(rng, value):
def safe_randint(rng, min_val, max_val, param_name=None):
"""Wrapper for randint() that validates integer arguments and provides context on failure."""
if not isinstance(min_val, int) or not isinstance(max_val, int):
context = f" (parameter: {param_name})" if param_name else ""
raise TypeError(
f"randint() requires integer arguments{context}. "
f"Got min={min_val} (type={type(min_val).__name__}), max={max_val} (type={type(max_val).__name__}). "
f"Fix: use int() in config file or add 'isUniform': True for float ranges."
)
return rng.randint(min_val, max_val)
def generate_normal_wt_parameters(rng, value, param_name=None):
"""Returns the value assigned the WiredTiger parameters (both eviction or table) based on the fields of the parameters in the config_fuzzer_wt_limits.py."""
if "choices" in value:
ret = rng.choice(value["choices"])
if "multiplier" in value:
ret *= value["multiplier"]
elif "min" in value and "max" in value:
ret = rng.randint(value["min"], value["max"])
ret = safe_randint(rng, value["min"], value["max"], param_name)
return ret
@ -96,7 +108,7 @@ def generate_eviction_configs(rng):
ret = generate_special_eviction_configs(rng, ret, params)
ret.update(
{
key: generate_normal_wt_parameters(rng, value)
key: generate_normal_wt_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params
}
@ -165,7 +177,7 @@ def generate_table_configs(rng):
ret.update(
{
key: generate_normal_wt_parameters(rng, value)
key: generate_normal_wt_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params
}
@ -222,7 +234,22 @@ def generate_encryption_config(rng: random.Random):
return ret
def generate_normal_mongo_parameters(rng, value):
def generate_runtime_mongod_parameter(rng, value, param_name):
"""Generate a runtime mongod parameter value, handling mongod-specific special cases.
This function is used for runtime fuzzing of mongod parameters and ensures consistent
behavior with startup generation for parameters that have special handling.
"""
# Special case: flowControlThresholdLagPercentage uses rng.random() which returns [0.0, 1.0)
# This matches the behavior in generate_flow_control_parameters() used at startup.
if param_name == "flowControlThresholdLagPercentage":
return rng.random()
# Default to normal generation for all other parameters
return generate_normal_mongo_parameters(rng, value, param_name)
def generate_normal_mongo_parameters(rng, value, param_name=None):
"""Returns the value assigned the mongod or mongos parameter based on the fields of the parameters in the config_fuzzer_limits.py."""
if "document" in value:
@ -231,7 +258,7 @@ def generate_normal_mongo_parameters(rng, value):
if "exclude_prob" in doc_value and rng.random() < doc_value["exclude_prob"]:
# Exclude this key from the document
continue
ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value)
ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value, doc_key)
elif "isUniform" in value:
ret = rng.uniform(value["min"], value["max"])
elif "isRandomizedChoice" in value:
@ -241,7 +268,7 @@ def generate_normal_mongo_parameters(rng, value):
elif "choices" in value:
ret = rng.choice(value["choices"])
elif "min" in value and "max" in value:
ret = rng.randint(value["min"], value["max"])
ret = safe_randint(rng, value["min"], value["max"], param_name)
if "multiplier" in value:
ret *= value["multiplier"]
elif "default" in value:
@ -338,10 +365,12 @@ def generate_flow_control_parameters(rng, ret, flow_control_params, params):
ret["enableFlowControl"] = rng.choice(params["enableFlowControl"]["choices"])
if ret["enableFlowControl"]:
for name in flow_control_params:
if name == "flowControlThresholdLagPercentage":
continue # Handled specially below with rng.random()
if "isUniform" in params[name]:
ret[name] = rng.uniform(params[name]["min"], params[name]["max"])
else:
ret[name] = rng.randint(params[name]["min"], params[name]["max"])
ret[name] = safe_randint(rng, params[name]["min"], params[name]["max"], name)
ret["flowControlThresholdLagPercentage"] = rng.random()
return ret
@ -398,7 +427,7 @@ def generate_mongod_parameters(rng):
# Range through all other parameters and assign the parameters based on the keys that are available or the parameter set lists defined above.
ret.update(
{
key: generate_normal_mongo_parameters(rng, value)
key: generate_normal_mongo_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params and key not in flow_control_params
}
@ -416,7 +445,7 @@ def generate_mongod_extra_configs(rng):
)
generated_config = {
key: generate_normal_mongo_parameters(rng, value)
key: generate_normal_mongo_parameters(rng, value, key)
for key, value in config_fuzzer_extra_configs["mongod"].items()
if not (value.get("enterprise_only", False) and "enterprise" not in config.MODULES)
}
@ -444,7 +473,7 @@ def generate_mongos_parameters(rng):
and not (val.get("enterprise_only", False) and "enterprise" not in config.MODULES)
}
return {key: generate_normal_mongo_parameters(rng, value) for key, value in params.items()}
return {key: generate_normal_mongo_parameters(rng, value, key) for key, value in params.items()}
def fuzz_mongod_set_parameters(seed, user_provided_params):

2
buildscripts/resmokelib/powercycle/setup/__init__.py
View File

@ -59,7 +59,7 @@ class SetUpEC2Instance(PowercycleCommand):
# Set up virtualenv on remote.
venv = powercycle_constants.VIRTUALENV_DIR
python = (
"/opt/mongodbtoolchain/v4/bin/python3"
"/opt/mongodbtoolchain/v5/bin/python3.13"
if "python" not in self.expansions
else self.expansions["python"]
)

2
buildscripts/resmokelib/run/__init__.py
View File

@ -2079,7 +2079,7 @@ class RunPlugin(PluginInterface):
dest="fuzz_mongod_configs",
help="Randomly chooses mongod parameters that were not specified.",
metavar="MODE",
choices=("normal"),
choices=("normal",),
)
mongodb_server_options.add_argument(

14
buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py
View File

@ -12,6 +12,7 @@ from pymongo.errors import OperationFailure
from buildscripts.resmokelib import config, errors
from buildscripts.resmokelib.generate_fuzz_config.mongo_fuzzer_configs import (
generate_normal_mongo_parameters,
generate_runtime_mongod_parameter,
)
from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
from buildscripts.resmokelib.testing.fixtures import replicaset, shardedcluster, standalone
@ -39,7 +40,7 @@ class RuntimeParametersState:
"""Encapsulates the runtime-state of a set of parameters we are fuzzing. Tracks the last time we set a parameter value and holds
the logic for generating new values."""
def __init__(self, spec, seed):
def __init__(self, spec, seed, generator_func=None):
# Initialize the runtime state of each parameter in the spec, including the lastSet time at now, so we start setting the parameters
# at appropriate intervals after the suite begins.
now = time.time()
@ -47,6 +48,10 @@ class RuntimeParametersState:
key: {**copy.deepcopy(value), "lastSet": now} for key, value in spec.items()
}
self._rng = random.Random(seed)
# Use provided generator function, or default to generate_normal_mongo_parameters for backward compatibility
self._generator_func = (
generator_func if generator_func is not None else generate_normal_mongo_parameters
)
def generate_parameters(self):
"""Returns a dictionary of what parameters should be set now, along with values to set them to, based on the last time the
@ -55,7 +60,7 @@ class RuntimeParametersState:
now = time.time()
for key, value in self._params.items():
if now - value["lastSet"] >= value["period"]:
ret[key] = generate_normal_mongo_parameters(self._rng, value)
ret[key] = self._generator_func(self._rng, value, key)
value["lastSet"] = now
return ret
@ -153,7 +158,10 @@ class FuzzRuntimeParameters(interface.Hook):
validate_runtime_parameter_spec(cluster_params)
# Construct the runtime state before the suite begins.
# The initial lastSet time of each parameter is the start time of the suite.
self._mongod_param_state = RuntimeParametersState(runtime_mongod_params, self._seed)
# Use generate_runtime_mongod_parameter for mongod params to handle special cases.
self._mongod_param_state = RuntimeParametersState(
runtime_mongod_params, self._seed, generate_runtime_mongod_parameter
)
self._mongos_param_state = RuntimeParametersState(runtime_mongos_params, self._seed)
self._cluster_param_state = RuntimeParametersState(cluster_params, self._seed)

10
buildscripts/tests/resmoke_end2end/README.md
View File

@ -1,11 +1,17 @@
First, activate the virtual environment:
```
mongodb_repo_root$ source python3-venv/bin/activate
```
- All end-to-end resmoke tests can be run via a resmoke suite itself:
```
mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 buildscripts/resmoke.py run --suites resmoke_end2end_tests
(python3-venv) mongodb_repo_root$ python buildscripts/resmoke.py run --suites resmoke_end2end_tests
```
- Finer grained control of tests can also be run with by invoking python's unittest main by hand. E.g:
```
mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file
(python3-venv) mongodb_repo_root$ python -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file
```

2
buildscripts/tests/resmoke_end2end/test_resmoke.py
View File

@ -933,7 +933,7 @@ class TestCoreAnalyzerFunctions(unittest.TestCase):
task_name = "test_tast_name"
execution = "0"
generated_task_name = get_generated_task_name(task_name, execution)
self.assertEquals(matches_generated_task_pattern(task_name, generated_task_name), execution)
self.assertEqual(matches_generated_task_pattern(task_name, generated_task_name), execution)
self.assertIsNone(matches_generated_task_pattern("not_same_task", generated_task_name))

2
docs/devcontainer/troubleshooting.md
View File

@ -421,7 +421,7 @@ For additional VS Code-specific troubleshooting, see:
```bash
rm -rf python3-venv
/opt/mongodbtoolchain/v5/bin/python3 -m venv python3-venv
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv
source python3-venv/bin/activate
poetry install --no-root --sync
```

8
etc/set_up_workstation.sh
View File

@ -56,7 +56,7 @@ setup_mongo_venv() {
# PYTHON_KEYRING_BACKEND is needed to make poetry install work
# See guide https://wiki.corp.mongodb.com/display/KERNEL/Virtual+Workstation
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
/opt/mongodbtoolchain/v4/bin/python3 -m venv python3-venv
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv
source ./python3-venv/bin/activate
POETRY_VIRTUALENVS_IN_PROJECT=true poetry install --no-root --sync
@ -86,17 +86,17 @@ setup_pipx() {
else
export PATH="$PATH:$HOME/.local/bin"
local venv_name="tmp-pipx-venv"
/opt/mongodbtoolchain/v4/bin/python3 -m venv $venv_name
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv $venv_name
# virtualenv doesn't like nounset
set +o nounset
source $venv_name/bin/activate
set -o nounset
python -m pip install --upgrade "pip<20.3"
python -m pip --disable-pip-version-check install "pip==25.3" "wheel==0.45.1"
python -m pip install pipx
pipx install pipx --python /opt/mongodbtoolchain/v4/bin/python3 --force
pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force
pipx ensurepath --force
set +o nounset

13
evergreen/prelude_python.sh
View File

@ -5,9 +5,16 @@ elif [ "$(uname)" = "Darwin" ]; then
python='/Library/Frameworks/Python.Framework/Versions/3.13/bin/python3'
echo "Executing on mac, setting python to ${python}"
else
if [ -f /opt/mongodbtoolchain/v5/bin/python3 ]; then
python="/opt/mongodbtoolchain/v5/bin/python3"
echo "Found python in v5 toolchain, setting python to ${python}"
# Check if v5 toolchain exists - it requires Python 3.13
if [ -d /opt/mongodbtoolchain/v5 ]; then
if [ -f /opt/mongodbtoolchain/v5/bin/python3.13 ]; then
python="/opt/mongodbtoolchain/v5/bin/python3.13"
echo "Found python 3.13 in v5 toolchain, setting python to ${python}"
else
echo "ERROR: v5 toolchain exists but Python 3.13 is not available at /opt/mongodbtoolchain/v5/bin/python3.13"
echo "The v5 toolchain requires Python 3.13. Please ensure python3.13 is installed in the toolchain."
return 1
fi
elif [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then
python="/opt/mongodbtoolchain/v4/bin/python3"
echo "Found python in v4 toolchain, setting python to ${python}"

2
evergreen/selinux_test_setup.sh
View File

@ -31,7 +31,7 @@ if ! sudo --non-interactive rpm --install --verbose --verbose --hash --nodeps "$
fi
# install packages needed by check_has_tag.py
PYTHON=/opt/mongodbtoolchain/v5/bin/python3
PYTHON=/opt/mongodbtoolchain/v5/bin/python3.13
if [[ (-f "$PYTHON" || -L "$PYTHON") && -x "$PYTHON" ]]; then
echo "==== Found python3 in $PYTHON"
$PYTHON -m pip install pyyaml

2
jstests/libs/python.js
View File

@ -20,7 +20,7 @@ export function getPython3Binary() {
}
const paths = [
"/opt/mongodbtoolchain/v5/bin/python3",
"/opt/mongodbtoolchain/v5/bin/python3.13",
"/opt/mongodbtoolchain/v4/bin/python3",
"/cygdrive/c/python/python313/python.exe",
"c:/python/python313/python.exe",

2
jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2
View File

@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2
View File

@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/histogramCE/plan_stability2
View File

@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/plan_stability2
View File

@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/samplingCE/plan_stability2
View File

@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

126
jstests/sharding/libs/proxy_protocol_server.py
View File

@ -8,13 +8,18 @@ The most recent source code is available on [GitHub][2].
The installed version is listed in the
"[tool.poetry.group.testing.dependencies]" section of `pyproject.toml`.
This wrapper adds two patches:
1. Logs "Now listening on [...]" when the server is ready
2. Calls server.close_clients() during shutdown for Python 3.13 compatibility
[1]: https://pypi.org/project/proxy-protocol/
[2]: https://github.com/icgood/proxy-protocol/
"""
import sys
from asyncio.base_events import BaseEventLoop
import proxyprotocol.server.main
from proxyprotocol.server.main import main
# We want to know when the proxy protocol server is ready to accept connections; so, we log to
@ -23,16 +28,123 @@ from proxyprotocol.server.main import main
# thing is to "monkey patch" the standard method
# [asyncio.base_events.BaseEventLoop.create_server][1] so that it logs after the server is created.
#
# Additionally, we store a reference to the server objects so they can be properly shut down.
#
# [1]: https://github.com/python/cpython/blob/5c19c5bac6abf3da97d1d9b80cfa16e003897096/Lib/asyncio/base_events.py#L1429
original_create_server = BaseEventLoop.create_server
_servers = []
_original_create_server = BaseEventLoop.create_server
async def monkeypatched_create_server(self, protocol_factory, host, port, *args, **kwargs):
result = await original_create_server(self, protocol_factory, host, port, *args, **kwargs)
print(f"Now listening on {host}:{port}")
return result
async def _patched_create_server(self, protocol_factory, host, port, *args, **kwargs):
server = await _original_create_server(self, protocol_factory, host, port, *args, **kwargs)
_servers.append(server)
print(f"Now listening on {host}:{port}", flush=True)
return server
BaseEventLoop.create_server = _patched_create_server
# Monkey-patch the library's run() function to add close_clients() call during shutdown.
#
# IMPORTANT: The code below is copied from proxyprotocol.server.main.run() v0.11.3
# (see python3-venv/lib/python3.*/site-packages/proxyprotocol/server/main.py)
# with minimal modifications for Python 3.13 compatibility.
#
# CHANGES FROM LIBRARY SOURCE:
# 1. Added server.close_clients() call in handle_signal() before forever.cancel()
# - Python 3.13 added Server.close_clients() to forcibly close active connections
# - Python 3.12+ fixed wait_closed() to correctly wait for connections
# - Library's original code relied on Python 3.10/3.11 bug where wait_closed()
# returned immediately even with active connections
# 2. Added logging to track signal handling and shutdown progress
#
_original_run = proxyprotocol.server.main.run
async def _patched_run(args):
import asyncio
import signal
from asyncio import CancelledError
from contextlib import AsyncExitStack
from functools import partial
from proxyprotocol.dnsbl import Dnsbl
from proxyprotocol.server import Address
from proxyprotocol.server.protocol import DownstreamProtocol, UpstreamProtocol
# --- BEGIN: Code copied from library's run() function ---
loop = asyncio.get_running_loop()
services = [(Address(source, server=True), Address(dest)) for (source, dest) in args.services]
buf_len = args.buf_len
dnsbl = Dnsbl.load(args.dnsbl, timeout=args.dnsbl_timeout)
new_server = partial(DownstreamProtocol, UpstreamProtocol, loop, buf_len, dnsbl)
servers = [
await loop.create_server(
partial(new_server, dest), source.host, source.port or 0, ssl=source.ssl
)
for source, dest in services
]
async with AsyncExitStack() as stack:
for server in servers:
await stack.enter_async_context(server)
forever = asyncio.gather(*[server.serve_forever() for server in servers])
# --- BEGIN MODIFICATION: Added proper shutdown sequence ---
def handle_signal():
print("Proxy server received shutdown signal", flush=True)
# Step 1: Stop accepting new connections
try:
for server in servers:
server.close()
print("Proxy server: stopped accepting new connections", flush=True)
except Exception as e:
print(f"Proxy server: error in close(): {e}", flush=True)
# Step 2: Close existing client connections gracefully
try:
for server in servers:
server.close_clients()
print("Proxy server: initiated graceful client close", flush=True)
except Exception as e:
print(f"Proxy server: error in close_clients(): {e}", flush=True)
# Step 3: Schedule abort as a safety fallback after 1 second
def abort_remaining():
print("Proxy server: aborting any remaining connections", flush=True)
try:
for server in servers:
server.abort_clients()
except Exception as e:
print(f"Proxy server: error in abort_clients(): {e}", flush=True)
loop.call_later(1.0, abort_remaining)
# CHANGE: Use proper shutdown sequence per Python 3.13 docs:
# 1. close() to stop accepting new connections
# 2. close_clients() to gracefully close existing connections
# 3. Schedule abort_clients() as fallback after timeout
forever.cancel()
# --- END MODIFICATION ---
loop.add_signal_handler(signal.SIGINT, handle_signal)
loop.add_signal_handler(signal.SIGTERM, handle_signal)
try:
await forever
except CancelledError:
pass
# --- END: Code copied from library's run() function ---
print("Proxy server shutdown complete", flush=True)
return 0
proxyprotocol.server.main.run = _patched_run
if __name__ == "__main__":
BaseEventLoop.create_server = monkeypatched_create_server
sys.exit(main())