mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-99783 generate bes_keywords in bazel wrapper script (#31689)

Browse Source 
GitOrigin-RevId: 8f2b1c0b5aa098f2302921d1772bff9e21f1b181
This commit is contained in:
Daniel Moody 2025-01-28 21:05:00 -06:00 committed by MongoDB Bot
parent ef5e3821b1
commit 7a03d9f50f
10 changed files with 151 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bazelrc
View File

@ -410,7 +410,7 @@ try-import %workspace%/.bazelrc.evergreen
try-import %workspace%/.bazelrc.xcode
# local default dev settings
try-import %workspace%/.bazelrc.workstation
try-import %workspace%/.bazelrc.common_bes
# local git version info
try-import %workspace%/.bazelrc.git

1
.gitignore vendored
View File

@ -281,6 +281,7 @@ buildozer
.bazelrc.git
.bazelrc.gitinfo
.bazelrc.workstation
.bazelrc.common_bes
.bazelrc.xcode
.bazelrc.bazelisk
*.bazel_info_for_ninja.txt

81
bazel/wrapper_hook/developer_bes_keywords.py Normal file
View File

@ -0,0 +1,81 @@
import base64
import hashlib
import json
import os
import socket
import git
def write_workstation_bazelrc(args):
workstation_file = ".bazelrc.common_bes"
existing_hash = ""
if os.path.exists(workstation_file):
with open(workstation_file) as f:
existing_hash = hashlib.md5(f.read().encode()).hexdigest()
try:
repo = git.Repo()
except Exception:
print(
"Unable to setup git repo, skipping workstation file generation. This will result in incomplete telemetry data being uploaded."
)
else:
try:
status = "clean" if repo.head.commit.diff(None) is None else "modified"
except Exception:
status = "Unknown"
try:
remote = repo.branches.master.repo.remote().url
except Exception:
try:
remote = repo.remotes[0].url
except Exception:
remote = "Unknown"
try:
branch = repo.active_branch.name
except Exception:
branch = "Unknown"
try:
commit = repo.commit("HEAD")
except Exception:
commit = "Unknown"
try:
reader = repo.config_reader()
user = reader.get_value("user", "email")
except Exception:
user = "Unknown"
if os.environ.get("CI") is not None:
user = os.environ.get("author_email", "Unknown")
try:
hostname = socket.gethostname()
except Exception:
hostname = "Unknown"
developer_build = os.environ.get("CI") is None
b64_cmd_line = base64.b64encode(json.dumps(args[1:]).encode()).decode()
bazelrc_contents = f"""\
# Generated file, do not modify
common --bes_keywords=developerBuild={developer_build}
common --bes_keywords=user_email={user}
common --bes_keywords=engflow:BuildScmRemote={remote}
common --bes_keywords=engflow:BuildScmBranch={branch}
common --bes_keywords=engflow:BuildScmRevision={commit}
common --bes_keywords=engflow:BuildScmStatus={status}
common --bes_keywords=rawCommandLineBase64={b64_cmd_line}
"""
if developer_build:
bazelrc_contents += f"common --bes_keywords=workstation={hostname}{os.linesep}"
current_hash = hashlib.md5(bazelrc_contents.encode()).hexdigest()
if existing_hash != current_hash:
print(f"Generating new {workstation_file} file...")
with open(workstation_file, "w") as f:
f.write(bazelrc_contents)

36
bazel/wrapper_hook/install_modules.py
View File

@ -29,6 +29,36 @@ def get_deps_dirs(deps):
yield f"{bazel_bin}/external/poetry/{dep}", dep
def add_module_to_path(poetry_dir, modules_added):
for module in poetry_dir.iterdir():
for dist_info in module.iterdir():
if str(dist_info).endswith(".dist-info"):
dirname = dist_info.parent
module = dirname.name
if module not in modules_added:
modules_added.add(module)
sys.path.append(str(dirname))
def setup_python_path():
tmp_dir = pathlib.Path(os.environ["Temp"] if platform.system() == "Windows" else "/tmp")
modules_added = set()
for out_dir in [
REPO_ROOT / "bazel-out",
tmp_dir / "compiledb-out",
]:
if out_dir.exists():
for child in out_dir.iterdir():
poetry_dir = child / "bin" / "external" / "poetry"
if poetry_dir.exists():
add_module_to_path(poetry_dir, modules_added)
poetry_dir = REPO_ROOT / "bazel-bin" / "external" / "poetry"
if poetry_dir.exists():
add_module_to_path(poetry_dir, modules_added)
def search_for_modules(deps, deps_installed, lockfile_changed=False):
deps_not_found = deps.copy()
wrapper_debug(f"deps_installed: {deps_installed}")
@ -46,7 +76,6 @@ def search_for_modules(deps, deps_installed, lockfile_changed=False):
wrapper_debug(f"found: {target_dir}")
deps_installed.append(dep)
deps_not_found.remove(dep)
sys.path.append(target_dir)
break
else:
os.chmod(target_dir, 0o777)
@ -76,7 +105,7 @@ def install_modules(bazel):
with open(lockfile_hash_file, "w") as f:
f.write(current_hash)
deps = ["retry"]
deps = ["retry", "gitpython"]
deps_installed = []
deps_needed = search_for_modules(
deps, deps_installed, lockfile_changed=old_hash != current_hash
@ -91,8 +120,9 @@ def install_modules(bazel):
if need_to_install:
subprocess.run(
[bazel, "build", "--config=local"] + ["@poetry//:install_" + dep for dep in deps_needed]
[bazel, "build", "--config=local"] + ["@poetry//:library_" + dep for dep in deps_needed]
)
deps_missing = search_for_modules(deps_needed, deps_installed)
if deps_missing:
raise Exception(f"Failed to install python deps {deps_missing}")
setup_python_path()

11
bazel/wrapper_hook/wrapper_hook.py
View File

@ -9,9 +9,7 @@ sys.path.append(str(REPO_ROOT))
# may be expecting certain stdout, always print to stderr.
sys.stdout = sys.stderr
from bazel.wrapper_hook.engflow_check import engflow_auth
from bazel.wrapper_hook.install_modules import install_modules
from bazel.wrapper_hook.plus_interface import test_runner_interface
from bazel.wrapper_hook.wrapper_debug import wrapper_debug
wrapper_debug(f"wrapper hook script is using {sys.executable}")
@ -20,14 +18,21 @@ wrapper_debug(f"wrapper hook script is using {sys.executable}")
def main():
install_modules(sys.argv[1])
from bazel.wrapper_hook.developer_bes_keywords import write_workstation_bazelrc
from bazel.wrapper_hook.engflow_check import engflow_auth
from bazel.wrapper_hook.plus_interface import test_runner_interface
engflow_auth(sys.argv)
write_workstation_bazelrc(sys.argv)
args = test_runner_interface(
sys.argv[1:], autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1"
)
os.chmod(os.environ.get("MONGO_BAZEL_WRAPPER_ARGS"), 0o644)
with open(os.environ.get("MONGO_BAZEL_WRAPPER_ARGS"), "w") as f:
f.write(" ".join(args))
f.write("\n".join(args))
f.write("\n")
if __name__ == "__main__":

2
etc/evergreen_yml_components/definitions.yml
View File

@ -1372,6 +1372,7 @@ functions:
binary: bash
env:
evergreen_remote_exec: ${evergreen_remote_exec|off}
author_email: ${author_email}
args:
- "src/evergreen/scons_compile.sh"
- command: expansions.update
@ -1463,6 +1464,7 @@ functions:
binary: bash
env:
evergreen_remote_exec: ${evergreen_remote_exec|off}
author_email: ${author_email}
args:
- "src/evergreen/bazel_compile.sh"

1
evergreen/generate_evergreen_bazelrc.sh
View File

@ -49,4 +49,3 @@ echo "common --bes_keywords=engflow:CiCdJobName=${task_name:?}" >> .bazelrc.ever
echo "common --bes_keywords=engflow:CiCdUri=${uri:?}" >> .bazelrc.evergreen
echo "common --bes_keywords=evg:project=${project:?}" >> .bazelrc.evergreen
echo "common --remote_upload_local_results=True" >> .bazelrc.evergreen
echo "common --workspace_status_command=./evergreen/engflow_workspace_status.sh" >> .bazelrc.evergreen

66
site_scons/site_tools/integrate_bazel.py
View File

@ -10,7 +10,6 @@ import queue
import shlex
import shutil
import signal
import socket
import stat
import subprocess
import sys
@ -23,7 +22,6 @@ from io import StringIO
from typing import Any, Dict, List, Set, Tuple
import distro
import git
import psutil
import requests
import SCons
@ -31,6 +29,7 @@ from retry import retry
from retry.api import retry_call
from SCons.Script import ARGUMENTS
from bazel.wrapper_hook.developer_bes_keywords import write_workstation_bazelrc
from buildscripts.install_bazel import install_bazel
from buildscripts.util.read_config import read_config_file
from evergreen.api import RetryingEvergreenApi
@ -1207,7 +1206,7 @@ def prefetch_toolchain(env, version):
def exists(env: SCons.Environment.Environment) -> bool:
# === Bazelisk ===
write_workstation_bazelrc()
write_workstation_bazelrc(sys.argv)
cleanup_gitinfo_bazelrc()
env.AddMethod(prefetch_toolchain, "PrefetchToolchain")
env.AddMethod(bazel_execroot, "BazelExecroot")
@ -1281,67 +1280,6 @@ def cleanup_gitinfo_bazelrc():
pass
def write_workstation_bazelrc():
if os.environ.get("CI") is None:
workstation_file = ".bazelrc.workstation"
existing_hash = ""
if os.path.exists(workstation_file):
with open(workstation_file) as f:
existing_hash = hashlib.md5(f.read().encode()).hexdigest()
try:
repo = git.Repo()
except Exception:
print(
"Unable to setup git repo, skipping workstation file generation. This will result in incomplete telemetry data being uploaded."
)
return
try:
status = "clean" if repo.head.commit.diff(None) is None else "modified"
except Exception:
status = "Unknown"
try:
hostname = socket.gethostname()
except Exception:
hostname = "Unknown"
try:
remote = repo.branches.master.repo.remote().url
except Exception:
try:
remote = repo.remotes[0].url
except Exception:
remote = "Unknown"
try:
branch = repo.active_branch.name
except Exception:
branch = "Unknown"
try:
commit = repo.commit("HEAD")
except Exception:
commit = "Unknown"
bazelrc_contents = f"""\
# Generated file, do not modify
common --bes_keywords=developerBuild=True
common --bes_keywords=workstation={hostname}
common --bes_keywords=engflow:BuildScmRemote={remote}
common --bes_keywords=engflow:BuildScmBranch={branch}
common --bes_keywords=engflow:BuildScmRevision={commit}
common --bes_keywords=engflow:BuildScmStatus={status}
"""
current_hash = hashlib.md5(bazelrc_contents.encode()).hexdigest()
if existing_hash != current_hash:
print(f"Generating new {workstation_file} file...")
with open(workstation_file, "w") as f:
f.write(bazelrc_contents)
def setup_bazel_env_vars() -> None:
# Set the JAVA_HOME directories for ppc64le and s390x since their bazel binaries are not compiled with a built-in JDK.
if platform.machine().lower() in {"ppc64le", "s390x"}:

18
tools/bazel
View File

@ -148,7 +148,7 @@ MONGO_BAZEL_WRAPPER_ARGS=$(mktemp)
MONGO_BAZEL_WRAPPER_ARGS=$MONGO_BAZEL_WRAPPER_ARGS \
MONGO_AUTOCOMPLETE_QUERY=$autocomplete_query \
$python $REPO_ROOT/bazel/wrapper_hook/wrapper_hook.py $bazel_real "$@"
>&2 $python $REPO_ROOT/bazel/wrapper_hook/wrapper_hook.py $bazel_real "$@"
if [[ $? != 0 ]]; then
if [[ ! -z "$CI" ]] || [[ $MONGO_BAZEL_WRAPPER_FALLBACK == 1 ]]; then
>&2 echo "wrapper script failed! falling back to normal bazel call..."
@ -157,21 +157,25 @@ if [[ $? != 0 ]]; then
exit $?
fi
fi
new_args=$(<$MONGO_BAZEL_WRAPPER_ARGS)
new_args=()
while IFS= read -r line; do
new_args+=("$line")
done < $MONGO_BAZEL_WRAPPER_ARGS
if [[ $MONGO_BAZEL_WRAPPER_DEBUG == 1 ]] && [[ $autocomplete_query == 0 ]]; then
wrapper_end_time="$(date -u +%s.%N)"
runtime=$(bc <<< "$wrapper_end_time - $wrapper_start_time")
runtime=$(printf "%0.3f" $runtime)
echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script input args: $@"
echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script new args: $new_args"
echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script took $runtime seconds"
>&2 echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script input args: $@"
>&2 echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script new args: ${new_args[@]@Q}"
>&2 echo "[WRAPPER_HOOK_DEBUG]: wrapper hook script took $runtime seconds"
fi
if [[ $autocomplete_query == 1 ]]; then
plus_targets=$(</tmp/mongo_autocomplete_plus_targets)
query_output=$("$bazel_real" $new_args)
query_output=$("${new_args[@]@Q}")
echo $query_output $plus_targets | tr " " "\n"
else
exec "$bazel_real" $new_args
exec "$bazel_real" "${new_args[@]}"
fi

19
tools/bazel.bat
View File

@ -19,7 +19,7 @@ if not exist "%python%" (
if %ERRORLEVEL% NEQ 0 (
if "%CI%"=="" if "%MONGO_BAZEL_WRAPPER_FALLBACK%"=="" exit %ERRORLEVEL%
echo wrapper script failed to install python! falling back to normal bazel call...
echo wrapper script failed to install python! falling back to normal bazel call... 1>&2
"%BAZEL_REAL%" %*
exit %ERRORLEVEL%
)
@ -32,14 +32,19 @@ SET STARTTIME=%TIME%
set "MONGO_BAZEL_WRAPPER_ARGS=%tmp%\bat~%RANDOM%.tmp"
echo "" > %MONGO_BAZEL_WRAPPER_ARGS%
%python% %REPO_ROOT%/bazel/wrapper_hook/wrapper_hook.py "%BAZEL_REAL%" %*
%python% %REPO_ROOT%/bazel/wrapper_hook/wrapper_hook.py "%BAZEL_REAL%" %* 1>&2
if %ERRORLEVEL% NEQ 0 (
if "%CI%"=="" if "%MONGO_BAZEL_WRAPPER_FALLBACK%"=="" exit %ERRORLEVEL%
echo wrapper script failed! falling back to normal bazel call...
echo wrapper script failed! falling back to normal bazel call... 1>&2
"%BAZEL_REAL%" %*
exit %ERRORLEVEL%
)
for /f "Tokens=* Delims=" %%x in ( %MONGO_BAZEL_WRAPPER_ARGS% ) do set "new_args=!new_args!%%x"
for /F "delims=" %%a in (%MONGO_BAZEL_WRAPPER_ARGS%) do (
set str="%%a"
call set str=!str: =^ !
set "new_args=!new_args! !str!"
)
del %MONGO_BAZEL_WRAPPER_ARGS%
REM Final Calculations
@ -64,9 +69,9 @@ IF %cc% lss 10 SET cc=0%cc%
SET DURATION=%mm%m and %ss%.%cc%s
if "%MONGO_BAZEL_WRAPPER_DEBUG%"=="1" (
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script input args: %*
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script new args: !new_args!
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script took %DURATION%
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script input args: %* 1>&2
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script new args: !new_args! 1>&2
ECHO [WRAPPER_HOOK_DEBUG]: wrapper hook script took %DURATION% 1>&2
)
"%BAZEL_REAL%" !new_args!