Compare commits
350 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9fbea53313 | ||
|
|
d067e66435 | ||
|
|
c382e41dde | ||
|
|
11931d956f | ||
|
|
36eb45c51b | ||
|
|
8eafdcf18d | ||
|
|
8fe8ccc913 | ||
|
|
d77c2c1b4a | ||
|
|
a803a0599c | ||
|
|
65bb179286 | ||
|
|
ad71b7376c | ||
|
|
2b1ff713b8 | ||
|
|
6ce4e14936 | ||
|
|
359b1260bd | ||
|
|
cc819502e7 | ||
|
|
87e057b66a | ||
|
|
1613813459 | ||
|
|
590a3ee8e5 | ||
|
|
58a1f4e2c7 | ||
|
|
f90fbdb88e | ||
|
|
bd287221f6 | ||
|
|
0b13f3b9d2 | ||
|
|
e6120da285 | ||
|
|
a5b88171e8 | ||
|
|
413475e158 | ||
|
|
1e0e62bb30 | ||
|
|
1de5943c67 | ||
|
|
dcc27418ff | ||
|
|
0b4a0e62c6 | ||
|
|
d4b6bdff20 | ||
|
|
dcc1dbb423 | ||
|
|
c03686cacb | ||
|
|
aea6ff0e69 | ||
|
|
4140d9d592 | ||
|
|
8ae6ea43ac | ||
|
|
e03c939eb6 | ||
|
|
ff55682afe | ||
|
|
88fd06161a | ||
|
|
e2fec6eb56 | ||
|
|
a6ab931c9a | ||
|
|
f55f2b459f | ||
|
|
5710e50f4f | ||
|
|
5f8dbc8713 | ||
|
|
fc727a014e | ||
|
|
63acb191e9 | ||
|
|
a84c0fa6ea | ||
|
|
9b2d350c40 | ||
|
|
1e575782ca | ||
|
|
676409152e | ||
|
|
290a3eb838 | ||
|
|
65de5d7e7a | ||
|
|
41edcff4b9 | ||
|
|
38ab0ee533 | ||
|
|
f2d2c748ae | ||
|
|
c2f432e02f | ||
|
|
e142ec7e13 | ||
|
|
9d1682cb70 | ||
|
|
c3a00304eb | ||
|
|
35bac269ba | ||
|
|
455604a26c | ||
|
|
5bc14fbeec | ||
|
|
3c3fb98ec8 | ||
|
|
7a9dabec9c | ||
|
|
45279cc75b | ||
|
|
701f29ddcc | ||
|
|
0d05f10f27 | ||
|
|
f2b0e65789 | ||
|
|
3efaf08dc7 | ||
|
|
bca052045b | ||
|
|
fa497b53f1 | ||
|
|
3c3ed51901 | ||
|
|
6ba665ccde | ||
|
|
72629252a3 | ||
|
|
a5b3b8aa5b | ||
|
|
cf18c7d602 | ||
|
|
c21ca3de19 | ||
|
|
6eba2d6cb5 | ||
|
|
e4fcbbf92a | ||
|
|
6a57b75a74 | ||
|
|
ee8722fdf6 | ||
|
|
77ef569881 | ||
|
|
ede7a0e5a2 | ||
|
|
7fc6b15695 | ||
|
|
d130eb1c50 | ||
|
|
a0fa6de4bf | ||
|
|
0c65ad8423 | ||
|
|
88b6828d24 | ||
|
|
ed46e9c63a | ||
|
|
82f22ff185 | ||
|
|
0639250f67 | ||
|
|
7ba4b31490 | ||
|
|
05e40124fa | ||
|
|
2c091b5ee4 | ||
|
|
5ba0504741 | ||
|
|
a2ccb4fe31 | ||
|
|
3a9ddf5e84 | ||
|
|
dd9d82c6da | ||
|
|
048a17c79f | ||
|
|
da25fbc10b | ||
|
|
23dd776926 | ||
|
|
29367986fc | ||
|
|
e035a53c64 | ||
|
|
655ebb0fa1 | ||
|
|
61d51036f9 | ||
|
|
9a7c840349 | ||
|
|
73dadb8ec4 | ||
|
|
db35cc3354 | ||
|
|
93f0027118 | ||
|
|
84c733352b | ||
|
|
c87b7b2632 | ||
|
|
48f10211e3 | ||
|
|
30e5202326 | ||
|
|
9ef9154b3c | ||
|
|
f3f6fd6173 | ||
|
|
19897d58bd | ||
|
|
e49562e8d6 | ||
|
|
e6a4d5e471 | ||
|
|
f0112f35f2 | ||
|
|
d2c8ab0fd0 | ||
|
|
af92bd90f3 | ||
|
|
4e977bf676 | ||
|
|
991e1109a4 | ||
|
|
97eec1df21 | ||
|
|
382b41c9d6 | ||
|
|
d8cc9dd86b | ||
|
|
f6411d0808 | ||
|
|
13fc9420f4 | ||
|
|
cb23179906 | ||
|
|
3145debd5e | ||
|
|
848dace7fc | ||
|
|
c6587d9cb9 | ||
|
|
19707dbf10 | ||
|
|
beca4ba8b2 | ||
|
|
6c6a5c3656 | ||
|
|
a863c9fadc | ||
|
|
aeb19e6da3 | ||
|
|
455c919165 | ||
|
|
bcf1ea4b28 | ||
|
|
ba235332c5 | ||
|
|
b559a3aab3 | ||
|
|
bb03ca79ce | ||
|
|
8ed73d25fd | ||
|
|
375aac847f | ||
|
|
0422ebf9a5 | ||
|
|
05303d777f | ||
|
|
8dad976ba2 | ||
|
|
74692da3cc | ||
|
|
e6d7124994 | ||
|
|
1bff549d1f | ||
|
|
a5aeb4d387 | ||
|
|
6707a21134 | ||
|
|
18e9ee9904 | ||
|
|
30bb760be0 | ||
|
|
989e1357f1 | ||
|
|
47e6b69dce | ||
|
|
d7a239f410 | ||
|
|
ae73e84ece | ||
|
|
7277584dd6 | ||
|
|
85729d2e4f | ||
|
|
dd09d25acc | ||
|
|
2ab5dad532 | ||
|
|
0b31cdade8 | ||
|
|
ce63cdf537 | ||
|
|
e28ebc56bf | ||
|
|
60c4ee2187 | ||
|
|
2995019970 | ||
|
|
397eb0bcb4 | ||
|
|
3c9c2f2747 | ||
|
|
48ae227c07 | ||
|
|
d4511d28ec | ||
|
|
889e1451b1 | ||
|
|
b7daf9f217 | ||
|
|
2e5d23ff56 | ||
|
|
58dca72b11 | ||
|
|
5981ffdf4f | ||
|
|
58fa16f921 | ||
|
|
e08e44656e | ||
|
|
96f7a84d3f | ||
|
|
aab726b545 | ||
|
|
d0e2146b8c | ||
|
|
2dc4bbb91a | ||
|
|
9f2273839d | ||
|
|
cef94c64c9 | ||
|
|
5410b0829b | ||
|
|
d528fdd2b4 | ||
|
|
3585bbee40 | ||
|
|
f53ae515e8 | ||
|
|
6466b2fb48 | ||
|
|
0facf06066 | ||
|
|
5dcc6f66f7 | ||
|
|
be44b88830 | ||
|
|
1d26946296 | ||
|
|
07d77e5061 | ||
|
|
02c6a29bbc | ||
|
|
1331603c57 | ||
|
|
e2640d348f | ||
|
|
27cc937065 | ||
|
|
d87902c241 | ||
|
|
28ebe34b8d | ||
|
|
7cf67a345e | ||
|
|
427194347f | ||
|
|
195c2b231f | ||
|
|
6be67289bb | ||
|
|
e0117e92da | ||
|
|
cc3025436f | ||
|
|
829bda19ce | ||
|
|
31b06936c6 | ||
|
|
4418d28265 | ||
|
|
3f23b647d6 | ||
|
|
d9814c89f0 | ||
|
|
221f4e012d | ||
|
|
228193a940 | ||
|
|
f913620192 | ||
|
|
a5eeb8939e | ||
|
|
7d97af9679 | ||
|
|
4be6f775aa | ||
|
|
9b7e9473b8 | ||
|
|
f69d078407 | ||
|
|
937e7161b8 | ||
|
|
56dbb1009d | ||
|
|
098a61f2a3 | ||
|
|
d29e20a451 | ||
|
|
9b4c631916 | ||
|
|
16bdaa6b69 | ||
|
|
2d5f499469 | ||
|
|
3facd5a845 | ||
|
|
40180020c2 | ||
|
|
9b4f468908 | ||
|
|
9e04e35005 | ||
|
|
0048f016e8 | ||
|
|
c2c5491b61 | ||
|
|
a8b0cb11c0 | ||
|
|
e32ffb05e4 | ||
|
|
7666e8fea6 | ||
|
|
298218ea4b | ||
|
|
6adf0f08a7 | ||
|
|
a165502772 | ||
|
|
3a37c48ae6 | ||
|
|
7af4a575d1 | ||
|
|
7f60803f1d | ||
|
|
33c0745c8a | ||
|
|
c9a651fe6a | ||
|
|
527bc506a2 | ||
|
|
138e6a6a1a | ||
|
|
e70e212e13 | ||
|
|
5b2863df43 | ||
|
|
aacfc54b8d | ||
|
|
94f72ca06b | ||
|
|
1fe2b708a9 | ||
|
|
d2abc347ac | ||
|
|
37ef6eff43 | ||
|
|
d8e609493f | ||
|
|
781e2aa218 | ||
|
|
f7f3d170c3 | ||
|
|
def54dbbaf | ||
|
|
da0fc3c1bc | ||
|
|
fc0636057c | ||
|
|
12a8c902f3 | ||
|
|
a020406ab9 | ||
|
|
fd4848d554 | ||
|
|
bb2dadf241 | ||
|
|
eb9af3a1da | ||
|
|
3c713fd09f | ||
|
|
6d56055c32 | ||
|
|
4211210a74 | ||
|
|
277257c1ed | ||
|
|
5c24c5da7c | ||
|
|
afcaf66903 | ||
|
|
4c2cee8153 | ||
|
|
4c99b3f706 | ||
|
|
d86f99ed7e | ||
|
|
a3edac5a71 | ||
|
|
f1496796d8 | ||
|
|
722d015456 | ||
|
|
2e536df639 | ||
|
|
a082c84cc9 | ||
|
|
f0b5ab5e38 | ||
|
|
aaab85f098 | ||
|
|
e72a10d4b8 | ||
|
|
9a81256d9c | ||
|
|
2191f30bf4 | ||
|
|
68b52d3865 | ||
|
|
07bfb33e54 | ||
|
|
d3f99007aa | ||
|
|
91f7205349 | ||
|
|
21e335fe3e | ||
|
|
627d3cecf7 | ||
|
|
a402b4d130 | ||
|
|
fa6bd7d221 | ||
|
|
1bc69b035c | ||
|
|
1b098d48d8 | ||
|
|
f1ef2c7dae | ||
|
|
85f50fb465 | ||
|
|
ed6a5c217e | ||
|
|
05c03f3e8a | ||
|
|
4751478dab | ||
|
|
e83005b5d4 | ||
|
|
57257d9ad6 | ||
|
|
ca8a87ef77 | ||
|
|
c1064af11a | ||
|
|
78711fa1b6 | ||
|
|
4e07a389ae | ||
|
|
c3f463cc86 | ||
|
|
c9876d0fa9 | ||
|
|
de9fef5886 | ||
|
|
faf12e9eae | ||
|
|
9c4b726869 | ||
|
|
b993867dce | ||
|
|
12dc03761c | ||
|
|
2d43d9c8df | ||
|
|
6ee0dcb9a0 | ||
|
|
65be29203b | ||
|
|
f0cb4751ff | ||
|
|
8f2c658842 | ||
|
|
b0f36602de | ||
|
|
a53de12948 | ||
|
|
0be07af779 | ||
|
|
64b6ddd767 | ||
|
|
67dc54bb93 | ||
|
|
b517e4c8a6 | ||
|
|
675d0b61d2 | ||
|
|
8387f4eb6a | ||
|
|
df068ab24f | ||
|
|
5cc52b5a90 | ||
|
|
9bd8c9267d | ||
|
|
213fce2e3e | ||
|
|
4909b2b67b | ||
|
|
91e5dcf79c | ||
|
|
812035b596 | ||
|
|
8a3d441ee9 | ||
|
|
0423879028 | ||
|
|
ef178b3ce5 | ||
|
|
dc5795b510 | ||
|
|
04b67683bf | ||
|
|
e033c93689 | ||
|
|
a1d1cbbc54 | ||
|
|
ec9b18ec4e | ||
|
|
deec3224eb | ||
|
|
d1722270a9 | ||
|
|
47bd14d734 | ||
|
|
34eee63cd5 | ||
|
|
db2e25a73a | ||
|
|
01575d0911 | ||
|
|
4f200e4524 | ||
|
|
a5fd97e029 | ||
|
|
e409c348a1 | ||
|
|
8cc63860eb | ||
|
|
97e19d9bc4 | ||
|
|
5b527bfc7b | ||
|
|
7a1b05c781 |
@ -5,6 +5,10 @@ src/third_party/protobuf/dist
|
|||||||
src/third_party/re2/dist
|
src/third_party/re2/dist
|
||||||
src/third_party/tcmalloc/dist
|
src/third_party/tcmalloc/dist
|
||||||
src/third_party/wiredtiger/dist
|
src/third_party/wiredtiger/dist
|
||||||
|
bazel/auto_header/.auto_header
|
||||||
|
src/mongo/db/modules/atlas/.auto_header
|
||||||
|
src/mongo/db/modules/enterprise/.auto_header
|
||||||
|
.git
|
||||||
|
|
||||||
# Ignore node_modules due to the following error
|
# Ignore node_modules due to the following error
|
||||||
# ERROR: in verify_node_modules_ignored:
|
# ERROR: in verify_node_modules_ignored:
|
||||||
|
|||||||
8
.bazelrc
8
.bazelrc
@ -88,6 +88,8 @@ common:macos --repo_env=LLVM_VERSION=19
|
|||||||
|
|
||||||
# Pin down the Microsoft Visual compiler. If you would like to use
|
# Pin down the Microsoft Visual compiler. If you would like to use
|
||||||
# the default compiler version installed in this host, comment the line.
|
# the default compiler version installed in this host, comment the line.
|
||||||
|
common:windows --repo_env=BAZEL_VS="C:/Program Files/Microsoft Visual Studio/2022/Professional"
|
||||||
|
common:windows --repo_env=BAZEL_VC="C:/Program Files/Microsoft Visual Studio/2022/Professional/VC"
|
||||||
common:windows --repo_env=BAZEL_VC_FULL_VERSION=14.31.31103
|
common:windows --repo_env=BAZEL_VC_FULL_VERSION=14.31.31103
|
||||||
|
|
||||||
# Default the Visual C Redistribution to v14.3 for Windows installer.
|
# Default the Visual C Redistribution to v14.3 for Windows installer.
|
||||||
@ -505,9 +507,6 @@ common:fission --remote_download_regex=.*\.dwo$
|
|||||||
# Avoid failing builds when BES metadata fails to upload.
|
# Avoid failing builds when BES metadata fails to upload.
|
||||||
common --bes_upload_mode=fully_async
|
common --bes_upload_mode=fully_async
|
||||||
|
|
||||||
# Default Mongo Version if a version is not specified.
|
|
||||||
common --define=MONGO_VERSION=8.2.0-alpha
|
|
||||||
|
|
||||||
# Default distmod if not specified.
|
# Default distmod if not specified.
|
||||||
common --define=MONGO_DISTMOD=""
|
common --define=MONGO_DISTMOD=""
|
||||||
|
|
||||||
@ -523,6 +522,9 @@ try-import %workspace%/.bazelrc.evergreen
|
|||||||
# local default dev settings
|
# local default dev settings
|
||||||
try-import %workspace%/.bazelrc.common_bes
|
try-import %workspace%/.bazelrc.common_bes
|
||||||
|
|
||||||
|
# local default dev settings
|
||||||
|
try-import %workspace%/.bazelrc.mongo_variables
|
||||||
|
|
||||||
# local git version info
|
# local git version info
|
||||||
try-import %workspace%/.bazelrc.git
|
try-import %workspace%/.bazelrc.git
|
||||||
|
|
||||||
|
|||||||
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -6,6 +6,7 @@
|
|||||||
external rules-lint-ignored=true
|
external rules-lint-ignored=true
|
||||||
**/*.tpl.h rules-lint-ignored=true
|
**/*.tpl.h rules-lint-ignored=true
|
||||||
**/*.tpl.cpp rules-lint-ignored=true
|
**/*.tpl.cpp rules-lint-ignored=true
|
||||||
|
rpm/*.spec rules-lint-ignored=true
|
||||||
src/mongo/bson/column/bson_column_compressed_data.inl rules-lint-ignored=true
|
src/mongo/bson/column/bson_column_compressed_data.inl rules-lint-ignored=true
|
||||||
*.idl linguist-language=yaml
|
*.idl linguist-language=yaml
|
||||||
|
|
||||||
|
|||||||
3219
.github/CODEOWNERS
vendored
3219
.github/CODEOWNERS
vendored
File diff suppressed because it is too large
Load Diff
35
.gitignore
vendored
35
.gitignore
vendored
@ -1,6 +1,7 @@
|
|||||||
/build
|
/build
|
||||||
/src/mongo/db/modules/*
|
/src/mongo/db/modules/*/
|
||||||
!/src/mongo/db/modules/enterprise/
|
!/src/mongo/db/modules/enterprise/
|
||||||
|
!/src/mongo/db/modules/atlas/
|
||||||
/.jsdbshell
|
/.jsdbshell
|
||||||
/.cache
|
/.cache
|
||||||
/.dbshell
|
/.dbshell
|
||||||
@ -11,7 +12,10 @@
|
|||||||
/perf.data
|
/perf.data
|
||||||
/perf.data.old
|
/perf.data.old
|
||||||
/massif.out.*
|
/massif.out.*
|
||||||
|
/merged_decls.json
|
||||||
/modules_dump.yaml
|
/modules_dump.yaml
|
||||||
|
/.tmp
|
||||||
|
!/.tmp/_placeholder_
|
||||||
venv
|
venv
|
||||||
|
|
||||||
*~
|
*~
|
||||||
@ -95,6 +99,7 @@ scratch
|
|||||||
/mongoshim*
|
/mongoshim*
|
||||||
/mongosniff*
|
/mongosniff*
|
||||||
/mongotrafficreader*
|
/mongotrafficreader*
|
||||||
|
/bin
|
||||||
|
|
||||||
# artifacts from db-contrib-tool
|
# artifacts from db-contrib-tool
|
||||||
/ksdecode*
|
/ksdecode*
|
||||||
@ -116,6 +121,9 @@ scratch
|
|||||||
/optimizer_gdb_test_program-*
|
/optimizer_gdb_test_program-*
|
||||||
/pretty_printer_test-*.py
|
/pretty_printer_test-*.py
|
||||||
/pretty_printer_test_program-*
|
/pretty_printer_test_program-*
|
||||||
|
/pretty_printer_test_launcher*
|
||||||
|
/install_compass*
|
||||||
|
/multiversion-config.yml
|
||||||
|
|
||||||
*.tgz
|
*.tgz
|
||||||
*.zip
|
*.zip
|
||||||
@ -224,6 +232,7 @@ selected_tests_config
|
|||||||
codereview.rc
|
codereview.rc
|
||||||
|
|
||||||
# Python venvs and virtualenvs
|
# Python venvs and virtualenvs
|
||||||
|
python-venv
|
||||||
python3-venv
|
python3-venv
|
||||||
python2-venv
|
python2-venv
|
||||||
|
|
||||||
@ -259,7 +268,7 @@ all_feature_flags.txt
|
|||||||
|
|
||||||
# generated by clang-tidy buildscripts
|
# generated by clang-tidy buildscripts
|
||||||
clang_tidy_fixes
|
clang_tidy_fixes
|
||||||
.clang-tidy
|
/.clang-tidy
|
||||||
|
|
||||||
#SCons runtime configuration
|
#SCons runtime configuration
|
||||||
scons_env.env
|
scons_env.env
|
||||||
@ -280,10 +289,6 @@ package-lock.json
|
|||||||
# jstestfuzz generated test directory
|
# jstestfuzz generated test directory
|
||||||
/jstestfuzz/
|
/jstestfuzz/
|
||||||
|
|
||||||
# docker volumes mapped in lint_fuzzer_sanity_patch.py
|
|
||||||
/jstestfuzzinput
|
|
||||||
/jstestfuzzoutput
|
|
||||||
|
|
||||||
# Bazel-related work
|
# Bazel-related work
|
||||||
# (Note that each string does NOT terminate with a '/' so that gitignore works properly)
|
# (Note that each string does NOT terminate with a '/' so that gitignore works properly)
|
||||||
bazel-*
|
bazel-*
|
||||||
@ -295,11 +300,16 @@ buildozer
|
|||||||
.bazelrc.gitinfo
|
.bazelrc.gitinfo
|
||||||
.bazelrc.workstation
|
.bazelrc.workstation
|
||||||
.bazelrc.common_bes
|
.bazelrc.common_bes
|
||||||
|
.bazelrc.mongo_variables
|
||||||
|
.bazelrc.mongo_version
|
||||||
.bazelrc.compiledb
|
.bazelrc.compiledb
|
||||||
.bazelrc.sync
|
.bazelrc.sync
|
||||||
|
.bazelrc.wrapper_hook
|
||||||
|
.bazelrc.engflow_creds
|
||||||
.compiledb
|
.compiledb
|
||||||
.bazelrc.xcode
|
.bazelrc.xcode
|
||||||
.bazelrc.bazelisk
|
.bazelrc.bazelisk
|
||||||
|
.bazelrc.exec_log_file
|
||||||
*.bazel_info_for_ninja.txt
|
*.bazel_info_for_ninja.txt
|
||||||
.ninja_last_command_line_targets.txt
|
.ninja_last_command_line_targets.txt
|
||||||
bazel/coverity/analysis/BUILD.bazel
|
bazel/coverity/analysis/BUILD.bazel
|
||||||
@ -308,8 +318,10 @@ src/mongo/db/modules/enterprise/autogenerated_targets/BUILD.bazel
|
|||||||
.bazel_header_list_cache
|
.bazel_header_list_cache
|
||||||
.bazel_real
|
.bazel_real
|
||||||
.mongo_checks_module_path
|
.mongo_checks_module_path
|
||||||
MODULE.bazel
|
**/MODULE.bazel.lock
|
||||||
MODULE.bazel.lock
|
!/MODULE.bazel.lock
|
||||||
|
.auto_header
|
||||||
|
|
||||||
# generated configs for external fixture suites
|
# generated configs for external fixture suites
|
||||||
docker_compose/
|
docker_compose/
|
||||||
|
|
||||||
@ -319,6 +331,7 @@ buildscripts/antithesis/base_images/mongo_binaries/bin
|
|||||||
buildscripts/antithesis/base_images/mongo_binaries/lib
|
buildscripts/antithesis/base_images/mongo_binaries/lib
|
||||||
buildscripts/antithesis/base_images/mongo_binaries/libvoidstar.so
|
buildscripts/antithesis/base_images/mongo_binaries/libvoidstar.so
|
||||||
buildscripts/antithesis/base_images/mongo_binaries/tsan.suppressions
|
buildscripts/antithesis/base_images/mongo_binaries/tsan.suppressions
|
||||||
|
buildscripts/antithesis/base_images/mongo_binaries/src
|
||||||
|
|
||||||
buildscripts/antithesis/base_images/workload/bin
|
buildscripts/antithesis/base_images/workload/bin
|
||||||
buildscripts/antithesis/base_images/workload/lib
|
buildscripts/antithesis/base_images/workload/lib
|
||||||
@ -341,3 +354,9 @@ etc/trimmed_system_perf.yml
|
|||||||
/version_expansions.yml
|
/version_expansions.yml
|
||||||
/engflow.*
|
/engflow.*
|
||||||
/.bazelrc.evergreen
|
/.bazelrc.evergreen
|
||||||
|
|
||||||
|
# Generated Typescript type declarations from IDLs
|
||||||
|
src/mongo/shell/*_gen.d.ts
|
||||||
|
|
||||||
|
# devcontainer uses pnpm which results in a large amount of environment files that aren't intended for git commits
|
||||||
|
.pnpm-store
|
||||||
|
|||||||
@ -40,6 +40,9 @@ version_expansions.yml
|
|||||||
# Ignore all formatting in third_party/*
|
# Ignore all formatting in third_party/*
|
||||||
src/third_party
|
src/third_party
|
||||||
|
|
||||||
|
# this file is automatically generated and conforms to formatting requirements
|
||||||
|
README.third_party.md
|
||||||
|
|
||||||
# Ignore anything in the build output directories
|
# Ignore anything in the build output directories
|
||||||
build
|
build
|
||||||
bazel-*
|
bazel-*
|
||||||
|
|||||||
14
BUILD.bazel
14
BUILD.bazel
@ -220,13 +220,13 @@ mongo_install(
|
|||||||
"//conditions:default": ["//src/mongo/db:mongod"],
|
"//conditions:default": ["//src/mongo/db:mongod"],
|
||||||
}),
|
}),
|
||||||
package_extract_name = select({
|
package_extract_name = select({
|
||||||
"//bazel/config:build_enterprise_linux_enabled": "mongodb-linux-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_linux_enabled": "mongodb-linux-{MONGO_ARCH}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//bazel/config:build_enterprise_linux_disabled": "mongodb-linux-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_linux_disabled": "mongodb-linux-{MONGO_ARCH}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//bazel/config:build_enterprise_windows_enabled": "mongodb-win32-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_windows_enabled": "mongodb-win32-{MONGO_ARCH}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//bazel/config:build_enterprise_windows_disabled": "mongodb-win32-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_windows_disabled": "mongodb-win32-{MONGO_ARCH}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//bazel/config:build_enterprise_mac_enabled": "mongodb-macos-{TARGET_CPU}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_mac_enabled": "mongodb-macos-{MONGO_ARCH}-enterprise-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//bazel/config:build_enterprise_mac_disabled": "mongodb-macos-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//bazel/config:build_enterprise_mac_disabled": "mongodb-macos-{MONGO_ARCH}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
"//conditions:default": "mongodb-{TARGET_CPU}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
"//conditions:default": "mongodb-{MONGO_ARCH}-{MONGO_DISTMOD}-{MONGO_VERSION}",
|
||||||
}),
|
}),
|
||||||
publish_debug_in_stripped = select({
|
publish_debug_in_stripped = select({
|
||||||
"@platforms//os:windows": True,
|
"@platforms//os:windows": True,
|
||||||
|
|||||||
1027
MODULE.bazel.lock
generated
Normal file
1027
MODULE.bazel.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
@ -68,9 +68,12 @@ filters:
|
|||||||
approvers:
|
approvers:
|
||||||
- 10gen/devprod-correctness
|
- 10gen/devprod-correctness
|
||||||
- 10gen/devprod-build
|
- 10gen/devprod-build
|
||||||
|
- "README.third_party.md":
|
||||||
|
approvers:
|
||||||
|
- 10gen/code-review-team-ssdlc
|
||||||
- "sbom.json":
|
- "sbom.json":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/server-security
|
- 10gen/code-review-team-ssdlc
|
||||||
- "MODULE.bazel*":
|
- "MODULE.bazel*":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/devprod-build
|
- 10gen/devprod-build
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
#  MongoDB README
|
#  MongoDB README
|
||||||
|
|
||||||
Welcome to MongoDB!
|
Welcome to MongoDB 8.2!
|
||||||
|
|
||||||
## Components
|
## Components
|
||||||
|
|
||||||
|
|||||||
@ -21,132 +21,101 @@ not authored by MongoDB, and has a license which requires reproduction,
|
|||||||
a notice will be included in
|
a notice will be included in
|
||||||
`THIRD-PARTY-NOTICES`.
|
`THIRD-PARTY-NOTICES`.
|
||||||
|
|
||||||
| Name | License | Vendored Version | Emits persisted data | Distributed in Release Binaries |
|
| Name | License | Vendored Version | Emits persisted data | Distributed in Release Binaries |
|
||||||
| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ---------------------------------------- | -------------------- | ------------------------------- |
|
| ---------------------------------------------------- | --------------------------------- | ---------------------------------------- | -------------------- | ------------------------------- |
|
||||||
| [Abseil] | Apache-2.0 | 20250512.1 | | ✗ |
|
| [Abseil Common Libraries (C++)] | Apache-2.0 | 20250512.1 | | ✗ |
|
||||||
| [arximboldi/immer] | BSL-1.0 | Unknown | | ✗ |
|
| [Asio C++ Library] | BSL-1.0 | 1.12.2 | | ✗ |
|
||||||
| [Asio C++ Library] | BSL-1.0 | 1.12.2 | | ✗ |
|
| [AWS SDK for C++] | Apache-2.0 | 1.11.471 | | ✗ |
|
||||||
| [aws-sdk - the AWS SDK client library] | Apache-2.0 | 1.11.471 | | ✗ |
|
| [benchmark] | Apache-2.0 | 1.5.2 | | |
|
||||||
| [benchmark] | Apache-2.0 | v1.5.2 | | |
|
| [Boost C++ Libraries] | BSL-1.0 | 1.88.0 | | ✗ |
|
||||||
| [Boost C++ Libraries - boost] | BSL-1.0 | 1.88.0 | | ✗ |
|
| [c-ares] | MIT | 1.27.0 | | ✗ |
|
||||||
| [c-ares] | MIT | 1.27.0 | | ✗ |
|
| [CRoaring] | Apache-2.0 OR MIT | 3.0.1 | | ✗ |
|
||||||
| [concurrencytest] | GPL-3.0-or-later | 0.1.2 | unknown | |
|
| [Cyrus SASL] | BSD-Attribution-HPND-disclaimer | 2.1.28 | | |
|
||||||
| [Cyrus SASL] | BSD-Attribution-HPND-disclaimer | 2.1.28 | unknown | |
|
| [fmt] | MIT | 11.1.3 | | ✗ |
|
||||||
| [dcleblanc/SafeInt] | MIT | 3.0.26 | | ✗ |
|
| [folly] | Apache-2.0 | 2023.12.25.00 | | ✗ |
|
||||||
| [derickr/timelib] | MIT | 2022.13 | | ✗ |
|
| [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ |
|
||||||
| [discover] | BSD-3-Clause | 0.4.0 | unknown | |
|
| [gRPC (C++)] | Apache-2.0 | 1.59.5 | | ✗ |
|
||||||
| [fmtlib/fmt] | MIT | 11.1.3 | | ✗ |
|
| [ICU4C - International Components for Unicode C/C++] | Unicode-3.0 | 57.1 | ✗ | ✗ |
|
||||||
| [folly] | Apache-2.0 | v2025.04.21.00 | | ✗ |
|
| [immer] | BSL-1.0 | 0b3aaf699b9d6f2e89f8e2b6d1221c307e02bda3 | | ✗ |
|
||||||
| [google-re2] | BSD-3-Clause | 2023-11-01 | | ✗ |
|
| [Intel® Decimal Floating-Point Math Library] | BSD-3-Clause | 2.0.1 | | ✗ |
|
||||||
| [google-snappy] | BSD-3-Clause | 1.1.10 | ✗ | ✗ |
|
| [JSON Schema Store] | Apache-2.0 | 6847cfc3a17a04a7664474212db50c627e1e3408 | | |
|
||||||
| [google/s2geometry] | Apache-2.0 | Unknown | ✗ | ✗ |
|
| [JSON-Schema-Test-Suite] | MIT | 728066f9c5c258ba3b1804a22a5b998f2ec77ec0 | | |
|
||||||
| [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ |
|
| [libmongocrypt] | Apache-2.0 | 1.14.0 | ✗ | ✗ |
|
||||||
| [grpc] | Apache-2.0 | 1.59.5 | | ✗ |
|
| [librdkafka - The Apache Kafka C/C++ library] | BSD-2-Clause | 2.0.2 | | ✗ |
|
||||||
| [ICU for C/C++ (ICU4C)] | BSD-3-Clause, MIT v2 with Ad Clause License, Public Domain, BSD-2-Clause | 57.1 | ✗ | ✗ |
|
| [LibTomCrypt] | Unlicense | 1.18.2 | ✗ | ✗ |
|
||||||
| [Intel Decimal Floating-Point Math Library] | BSD-3-Clause | v2.0 U1 | | ✗ |
|
| [libunwind] | MIT | 1.8.1 | | ✗ |
|
||||||
| [jbeder/yaml-cpp] | MIT | 0.6.3 | | ✗ |
|
| [linenoise] | BSD-2-Clause | 6cdc775807e57b2c3fd64bd207814f8ee1fe35f3 | | ✗ |
|
||||||
| [JSON-Schema-Test-Suite] | Unknown License | Unknown | | |
|
| [MongoDB C Driver] | Apache-2.0 | 1.28.1 | ✗ | ✗ |
|
||||||
| [libmongocrypt] | Apache-2.0 | 1.14.0 | ✗ | ✗ |
|
| [Mozilla Firefox ESR] | MPL-2.0 | 128.11.0esr | | ✗ |
|
||||||
| [librdkafka - the Apache Kafka C/C++ client library] | BSD-3-Clause, Xmlproc License, ISC, MIT, Public Domain, Zlib, BSD-2-Clause, Andreas Stolcke License | 2.0.2 | | ✗ |
|
| [MurmurHash3] | Public Domain | a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb | | ✗ |
|
||||||
| [LibTomCrypt] | WTFPL, Public Domain | 1.18.2 | ✗ | ✗ |
|
| [nlohmann/json] | MIT | 3.11.3 | ✗ | |
|
||||||
| [libunwind/libunwind] | MIT | v1.8.1 | | ✗ |
|
| [node] | ISC | 22.1.0 | | |
|
||||||
| [linenoise] | BSD-2-Clause | Unknown | | ✗ |
|
| [opentelemetry-cpp] | Apache-2.0 | 1.17.0 | ✗ | |
|
||||||
| [MongoDB C Driver] | Apache-2.0 | 1.28.1 | ✗ | ✗ |
|
| [opentelemetry-proto] | Apache-2.0 | 1.3.2 | ✗ | |
|
||||||
| [Mozilla Firefox] | MPL-2.0 | 128.11.0esr | unknown | ✗ |
|
| [PCRE2 - Perl-Compatible Regular Expressions] | BSD-3-Clause WITH PCRE2-exception | 10.40 | | ✗ |
|
||||||
| [nlohmann-json] | MIT | 3.11.3 | ✗ | |
|
| [Protobuf] | BSD-3-Clause | v25.0 | | ✗ |
|
||||||
| [nlohmann.json.decomposed] | MIT | 3.10.5 | unknown | |
|
| [pypi/ocspbuilder] | MIT | 0.10.2 | | |
|
||||||
| [node] | ISC | 22.1.0 | unknown | |
|
| [pypi/ocspresponder] | Apache-2.0 | 0.5.0 | | |
|
||||||
| [ocspbuilder] | MIT | 0.10.2 | | |
|
| [re2] | BSD-3-Clause | 2023-11-01 | | ✗ |
|
||||||
| [ocspresponder] | Apache-2.0 | 0.5.0 | | |
|
| [S2 Geometry Library] | Apache-2.0 | a25c502bda9d7e0274b9e2b7825fbddf13cc0306 | ✗ | ✗ |
|
||||||
| [opentelemetry-cpp] | Apache-2.0 | 1.17 | ✗ | |
|
| [SafeInt] | MIT | 3.0.26 | | ✗ |
|
||||||
| [opentelemetry-proto] | Apache-2.0 | 1.3.2 | ✗ | |
|
| [snappy] | BSD-3-Clause | 1.1.10 | ✗ | ✗ |
|
||||||
| [PCRE2] | BSD-3-Clause, Public Domain | 10.40 | | ✗ |
|
| [Snowball Stemming Algorithms (libstemmer)] | BSD-3-Clause | 1.0.0 | ✗ | ✗ |
|
||||||
| [Protobuf] | BSD-3-Clause | v4.25.0 | | ✗ |
|
| [tcmalloc] | Apache-2.0 | f3b20f9a07e175c5d897df7b49d9830d4efa6110 | | ✗ |
|
||||||
| [pyiso8601] | MIT | 2.1.0 | unknown | |
|
| [timelib] | MIT | 2022.13 | | ✗ |
|
||||||
| [RoaringBitmap/CRoaring] | Unknown License | v3.0.1 | | ✗ |
|
| [Unicode Character Database] | Unicode-DFS-2016 | 8.0.0 | ✗ | ✗ |
|
||||||
| [SchemaStore/schemastore] | Apache-2.0 | Unknown | | |
|
| [valgrind.h] | BSD-4-Clause | 093bef43d69236287ccc748591c9560a71181b0a | | ✗ |
|
||||||
| [smhasher] | Unknown License | Unknown | unknown | ✗ |
|
| [WiredTiger] | GPL-2.0-only OR GPL-3.0-only | 12.0.0 | ✗ | ✗ |
|
||||||
| [Snowball Stemming Algorithms] | BSD-3-Clause | 7b264ffa0f767c579d052fd8142558dc8264d795 | ✗ | ✗ |
|
| [yaml-cpp] | MIT | 0.6.3 | | ✗ |
|
||||||
| [subunit] | BSD-3-Clause, Apache-2.0 | 1.4.4 | unknown | |
|
| [zlib] | Zlib | 1.3.1 | ✗ | ✗ |
|
||||||
| [tcmalloc] | Apache-2.0 | 20230227-snapshot-093ba93c | | ✗ |
|
| [Zstandard (zstd)] | BSD-3-Clause OR GPL-2.0-only | 1.5.5 | ✗ | ✗ |
|
||||||
| [testing-cabal/extras] | MIT | 0.0.3 | unknown | |
|
|
||||||
| [testscenarios] | BSD-3-Clause, Apache-2.0 | 0.4 | unknown | |
|
|
||||||
| [testtools] | MIT | 2.7.1 | unknown | |
|
|
||||||
| [unicode-data] | Unicode-DFS-2016 | 8.0 | ✗ | ✗ |
|
|
||||||
| [valgrind] | GPL-2.0-or-later | Unknown | | ✗ |
|
|
||||||
| [zlib] | Zlib | v1.3.1 | ✗ | ✗ |
|
|
||||||
| [zstd] | BSD-3-Clause, GPL-2.0-or-later | 1.5.5 | ✗ | ✗ |
|
|
||||||
|
|
||||||
[Abseil]: https://github.com/abseil/abseil-cpp
|
[AWS SDK for C++]: https://github.com/aws/aws-sdk-cpp.git
|
||||||
[Asio C++ Library]: https://github.com/chriskohlhoff/asio
|
[Abseil Common Libraries (C++)]: https://github.com/abseil/abseil-cpp.git
|
||||||
[Boost C++ Libraries - boost]: http://www.boost.org/
|
[Asio C++ Library]: https://github.com/chriskohlhoff/asio.git
|
||||||
[Cyrus SASL]: https://www.cyrusimap.org/sasl/
|
[Boost C++ Libraries]: https://github.com/boostorg/boost.git
|
||||||
[ICU for C/C++ (ICU4C)]: http://site.icu-project.org/download/
|
[CRoaring]: https://github.com/roaringbitmap/croaring.git
|
||||||
[Intel Decimal Floating-Point Math Library]: https://software.intel.com/en-us/articles/intel-decimal-floating-point-math-library
|
[Cyrus SASL]: https://github.com/cyrusimap/cyrus-sasl.git
|
||||||
[JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite
|
[ICU4C - International Components for Unicode C/C++]: https://github.com/unicode-org/icu.git
|
||||||
[LibTomCrypt]: https://github.com/libtom/libtomcrypt/releases
|
[Intel® Decimal Floating-Point Math Library]: https://www.netlib.org/misc/intel/
|
||||||
[MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver
|
[JSON Schema Store]: https://github.com/schemastore/schemastore.git
|
||||||
[Mozilla Firefox]: https://www.mozilla.org/en-US/security/known-vulnerabilities/firefox-esr
|
[JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite.git
|
||||||
[PCRE2]: http://www.pcre.org/
|
[LibTomCrypt]: https://github.com/libtom/libtomcrypt.git
|
||||||
[Protobuf]: https://github.com/protocolbuffers/protobuf
|
[MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver.git
|
||||||
[RoaringBitmap/CRoaring]: https://github.com/RoaringBitmap/CRoaring
|
[Mozilla Firefox ESR]: https://github.com/mozilla-firefox/firefox.git
|
||||||
[SchemaStore/schemastore]: https://www.schemastore.org/json/
|
[MurmurHash3]: https://github.com/aappleby/smhasher/blob/a6bd3ce/
|
||||||
[Snowball Stemming Algorithms]: https://github.com/snowballstem/snowball
|
[PCRE2 - Perl-Compatible Regular Expressions]: https://github.com/pcre2project/pcre2.git
|
||||||
[arximboldi/immer]: https://github.com/arximboldi/immer
|
[Protobuf]: https://github.com/protocolbuffers/protobuf.git
|
||||||
[aws-sdk - the AWS SDK client library]: https://github.com/aws/aws-sdk-cpp
|
[S2 Geometry Library]: https://github.com/google/s2geometry.git
|
||||||
[benchmark]: https://github.com/google/benchmark
|
[SafeInt]: https://github.com/dcleblanc/safeint.git
|
||||||
[c-ares]: https://c-ares.org/
|
[Snowball Stemming Algorithms (libstemmer)]: http://github.com/snowballstem/snowball.git
|
||||||
[concurrencytest]: https://pypi.org/project/concurrencytest/
|
[Unicode Character Database]: https://www.unicode.org/Public/8.0.0/
|
||||||
[dcleblanc/SafeInt]: https://github.com/dcleblanc/SafeInt
|
[WiredTiger]: https://github.com/wiredtiger/wiredtiger.git
|
||||||
[derickr/timelib]: https://github.com/derickr/timelib
|
[Zstandard (zstd)]: https://github.com/facebook/zstd.git
|
||||||
[discover]: https://pypi.org/project/discover/
|
[benchmark]: https://github.com/google/benchmark.git
|
||||||
[fmtlib/fmt]: http://fmtlib.net/
|
[c-ares]: https://github.com/c-ares/c-ares.git
|
||||||
[folly]: https://github.com/facebook/folly
|
[fmt]: https://github.com/fmtlib/fmt.git
|
||||||
[google-re2]: https://github.com/google/re2
|
[folly]: https://github.com/facebook/folly.git
|
||||||
[google-snappy]: https://github.com/google/snappy/releases
|
[gRPC (C++)]: https://github.com/grpc/grpc.git
|
||||||
[google/s2geometry]: https://github.com/google/s2geometry
|
[gperftools]: https://github.com/gperftools/gperftools.git
|
||||||
[gperftools]: https://github.com/gperftools/gperftools
|
[immer]: https://github.com/arximboldi/immer.git
|
||||||
[grpc]: https://github.com/grpc/grpc
|
[libmongocrypt]: https://github.com/mongodb/libmongocrypt.git
|
||||||
[jbeder/yaml-cpp]: https://github.com/jbeder/yaml-cpp/releases
|
[librdkafka - The Apache Kafka C/C++ library]: https://github.com/confluentinc/librdkafka.git
|
||||||
[libmongocrypt]: https://github.com/mongodb/libmongocrypt
|
[libunwind]: https://github.com/libunwind/libunwind.git
|
||||||
[librdkafka - the Apache Kafka C/C++ client library]: https://github.com/confluentinc/librdkafka
|
|
||||||
[libunwind/libunwind]: http://www.github.com/libunwind/libunwind
|
|
||||||
[linenoise]: https://github.com/antirez/linenoise
|
[linenoise]: https://github.com/antirez/linenoise
|
||||||
[nlohmann-json]: https://github.com/open-telemetry/opentelemetry-proto
|
[nlohmann/json]: https://github.com/nlohmann/json.git
|
||||||
[nlohmann.json.decomposed]: https://www.nuget.org/packages/nlohmann.json.decomposed
|
[node]: https://nodejs.org/
|
||||||
[node]: https://nodejs.org/en/blog/release
|
[opentelemetry-cpp]: https://github.com/open-telemetry/opentelemetry-cpp.git
|
||||||
[ocspbuilder]: https://github.com/wbond/ocspbuilder
|
|
||||||
[ocspresponder]: https://github.com/threema-ch/ocspresponder
|
|
||||||
[opentelemetry-cpp]: https://github.com/open-telemetry/opentelemetry-cpp/
|
|
||||||
[opentelemetry-proto]: https://github.com/open-telemetry/opentelemetry-proto
|
[opentelemetry-proto]: https://github.com/open-telemetry/opentelemetry-proto
|
||||||
[pyiso8601]: https://pypi.org/project/iso8601/
|
[pypi/ocspbuilder]: https://pypi.org/project/ocspbuilder/
|
||||||
[smhasher]: https://github.com/aappleby/smhasher/blob/a6bd3ce/
|
[pypi/ocspresponder]: https://pypi.org/project/ocspresponder/
|
||||||
[subunit]: https://github.com/testing-cabal/subunit
|
[re2]: https://github.com/google/re2.git
|
||||||
[tcmalloc]: https://github.com/google/tcmalloc
|
[snappy]: https://github.com/google/tcmalloc.git
|
||||||
[testing-cabal/extras]: https://github.com/testing-cabal/extras
|
[tcmalloc]: https://github.com/google/tcmalloc.git
|
||||||
[testscenarios]: https://pypi.org/project/testscenarios/
|
[timelib]: https://github.com/derickr/timelib.git
|
||||||
[testtools]: https://github.com/testing-cabal/testtools
|
[valgrind.h]: https://sourceware.org/git/valgrind.git
|
||||||
[unicode-data]: http://www.unicode.org/versions/enumeratedversions.html
|
[yaml-cpp]: https://github.com/jbeder/yaml-cpp.git
|
||||||
[valgrind]: http://valgrind.org/downloads/current.html
|
[zlib]: https://zlib.net/fossils/
|
||||||
[zlib]: https://zlib.net/
|
|
||||||
[zstd]: https://github.com/facebook/zstd
|
|
||||||
|
|
||||||
## WiredTiger Vendored Test Libraries
|
|
||||||
|
|
||||||
The following Python libraries are transitively included by WiredTiger,
|
|
||||||
and are used by that component for testing. They don't appear in
|
|
||||||
released binary artifacts.
|
|
||||||
|
|
||||||
| Name |
|
|
||||||
| ------------------------ |
|
|
||||||
| concurrencytest |
|
|
||||||
| discover |
|
|
||||||
| nlohmann.json.decomposed |
|
|
||||||
| pyiso8601 |
|
|
||||||
| subunit |
|
|
||||||
| testing-cabal/extras |
|
|
||||||
| testscenarios |
|
|
||||||
| testtools |
|
|
||||||
|
|
||||||
## Dynamically Linked Libraries
|
## Dynamically Linked Libraries
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
|
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
|
||||||
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
|
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
|
||||||
load("//bazel/config:configs.bzl", "sdkroot_provider")
|
load("//bazel/config:configs.bzl", "sdkroot_provider")
|
||||||
load("//bazel:mongo_src_rules.bzl", "write_target")
|
load("//bazel:utils.bzl", "write_target")
|
||||||
|
|
||||||
def generate_config_header_impl(ctx):
|
def generate_config_header_impl(ctx):
|
||||||
cc_toolchain = find_cpp_toolchain(ctx)
|
cc_toolchain = find_cpp_toolchain(ctx)
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
load("//bazel:mongo_src_rules.bzl", "write_target")
|
load("//bazel:utils.bzl", "write_target")
|
||||||
|
|
||||||
def render_template_impl(ctx):
|
def render_template_impl(ctx):
|
||||||
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
|
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
py_binary(
|
py_binary(
|
||||||
name = "generate_coverity_command",
|
name = "generate_coverity_targets",
|
||||||
srcs = ["generate_coverity_command.py"],
|
srcs = ["generate_coverity_targets.py"],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|||||||
@ -17,45 +17,27 @@ bazel_cache = os.path.expanduser(args.bazel_cache)
|
|||||||
# the cc_library and cc_binaries in our build. There is not a good way from
|
# the cc_library and cc_binaries in our build. There is not a good way from
|
||||||
# within the build to get all those targets, so we will generate the list via query
|
# within the build to get all those targets, so we will generate the list via query
|
||||||
# https://sig-product-docs.synopsys.com/bundle/coverity-docs/page/coverity-analysis/topics/building_with_bazel.html#build_with_bazel
|
# https://sig-product-docs.synopsys.com/bundle/coverity-docs/page/coverity-analysis/topics/building_with_bazel.html#build_with_bazel
|
||||||
proc = subprocess.run(
|
cmd = (
|
||||||
[
|
[
|
||||||
bazel_executable,
|
bazel_executable,
|
||||||
bazel_cache,
|
bazel_cache,
|
||||||
"aquery",
|
"aquery",
|
||||||
]
|
]
|
||||||
+ bazel_cmd_args
|
+ bazel_cmd_args
|
||||||
+ [args.bazel_query],
|
+ [args.bazel_query]
|
||||||
|
)
|
||||||
|
print(f"Running command: {cmd}")
|
||||||
|
proc = subprocess.run(
|
||||||
|
cmd,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE,
|
stderr=subprocess.PIPE,
|
||||||
text=True,
|
text=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
print(proc.stderr)
|
||||||
|
|
||||||
targets = set()
|
targets = set()
|
||||||
for line in proc.stdout.splitlines():
|
with open("coverity_targets.list", "w") as f:
|
||||||
if line.startswith(" Target: "):
|
for line in proc.stdout.splitlines():
|
||||||
targets.add(line.split()[-1])
|
if line.startswith(" Target: "):
|
||||||
|
f.write(line.split()[-1] + "\n")
|
||||||
|
|
||||||
enterprise_coverity_dir = os.path.join("src", "mongo", "db", "modules", "enterprise", "coverity")
|
|
||||||
os.makedirs(enterprise_coverity_dir, exist_ok=True)
|
|
||||||
with open(os.path.join(enterprise_coverity_dir, "BUILD.bazel"), "w") as buildfile:
|
|
||||||
buildfile.write("""\
|
|
||||||
load("@rules_coverity//coverity:defs.bzl", "cov_gen_script")
|
|
||||||
cov_gen_script(
|
|
||||||
name="enterprise_coverity_build",
|
|
||||||
testonly=True,
|
|
||||||
tags=["coverity"],
|
|
||||||
deps=[
|
|
||||||
""")
|
|
||||||
for target in targets:
|
|
||||||
buildfile.write(
|
|
||||||
"""\
|
|
||||||
"%s",
|
|
||||||
"""
|
|
||||||
% target
|
|
||||||
)
|
|
||||||
|
|
||||||
buildfile.write("""\
|
|
||||||
],
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
@ -35,6 +35,7 @@ format_multirun(
|
|||||||
graphql = "//:prettier",
|
graphql = "//:prettier",
|
||||||
html = "//:prettier",
|
html = "//:prettier",
|
||||||
markdown = "//:prettier",
|
markdown = "//:prettier",
|
||||||
|
python = "@aspect_rules_lint//format:ruff",
|
||||||
shell = "@shfmt//:shfmt",
|
shell = "@shfmt//:shfmt",
|
||||||
sql = "//:prettier",
|
sql = "//:prettier",
|
||||||
starlark = "@buildifier_prebuilt//:buildifier",
|
starlark = "@buildifier_prebuilt//:buildifier",
|
||||||
|
|||||||
@ -33,6 +33,9 @@ load(
|
|||||||
)
|
)
|
||||||
load("@local_host_values//:local_host_values_set.bzl", "NUM_CPUS")
|
load("@local_host_values//:local_host_values_set.bzl", "NUM_CPUS")
|
||||||
load("@evergreen_variables//:evergreen_variables.bzl", "UNSAFE_COMPILE_VARIANT", "UNSAFE_VERSION_ID")
|
load("@evergreen_variables//:evergreen_variables.bzl", "UNSAFE_COMPILE_VARIANT", "UNSAFE_VERSION_ID")
|
||||||
|
load("//bazel/toolchains/cc/mongo_windows:mongo_windows_cc_toolchain_config.bzl", "MIN_VER_MAP")
|
||||||
|
load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
|
||||||
|
load("//bazel/config:generate_config_header.bzl", "generate_config_header")
|
||||||
|
|
||||||
# These will throw an error if the following condition is not met:
|
# These will throw an error if the following condition is not met:
|
||||||
# (libunwind == on && os == linux) || libunwind == off || libunwind == auto
|
# (libunwind == on && os == linux) || libunwind == off || libunwind == auto
|
||||||
@ -1117,27 +1120,6 @@ idl_generator_rule = rule(
|
|||||||
fragments = ["py"],
|
fragments = ["py"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def write_target_impl(ctx):
|
|
||||||
out = ctx.actions.declare_file(ctx.label.name + ".gen_source_list")
|
|
||||||
ctx.actions.write(
|
|
||||||
out,
|
|
||||||
"//" + ctx.label.package + ":" + ctx.attr.target_name,
|
|
||||||
)
|
|
||||||
return [
|
|
||||||
DefaultInfo(
|
|
||||||
files = depset([out]),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
write_target = rule(
|
|
||||||
write_target_impl,
|
|
||||||
attrs = {
|
|
||||||
"target_name": attr.string(
|
|
||||||
doc = "the name of the target to record",
|
|
||||||
),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def idl_generator(name, tags = [], **kwargs):
|
def idl_generator(name, tags = [], **kwargs):
|
||||||
idl_generator_rule(
|
idl_generator_rule(
|
||||||
name = name,
|
name = name,
|
||||||
@ -1464,3 +1446,155 @@ def mongo_cc_fuzzer_test(
|
|||||||
exec_properties = exec_properties,
|
exec_properties = exec_properties,
|
||||||
**kwargs
|
**kwargs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _compute_win_defines(ver):
|
||||||
|
if ver not in MIN_VER_MAP:
|
||||||
|
fail("Unsupported windows_version_minimal=%r; supported: %r" %
|
||||||
|
(ver, sorted(MIN_VER_MAP.keys())))
|
||||||
|
m = MIN_VER_MAP[ver]
|
||||||
|
return [
|
||||||
|
"_WIN32_WINNT=%s" % m["win"],
|
||||||
|
"BOOST_USE_WINAPI_VERSION=%s" % m["win"],
|
||||||
|
"NTDDI_VERSION=%s" % m["ddi"],
|
||||||
|
]
|
||||||
|
|
||||||
|
def _rc_impl(ctx):
|
||||||
|
# No-op on non-Windows
|
||||||
|
if not ctx.target_platform_has_constraint(
|
||||||
|
ctx.attr._os_windows[platform_common.ConstraintValueInfo],
|
||||||
|
):
|
||||||
|
li = cc_common.create_linker_input(
|
||||||
|
owner = ctx.label,
|
||||||
|
user_link_flags = [],
|
||||||
|
additional_inputs = depset(), # and declare it as a link input
|
||||||
|
)
|
||||||
|
new_cc_shared_info = CcSharedLibraryInfo(
|
||||||
|
dynamic_deps = depset(),
|
||||||
|
exports = [],
|
||||||
|
linker_input = li,
|
||||||
|
link_once_static_libs = {},
|
||||||
|
)
|
||||||
|
|
||||||
|
return [DefaultInfo(files = depset([])), CcInfo(), new_cc_shared_info]
|
||||||
|
|
||||||
|
new_cc_shared_info = CcSharedLibraryInfo(
|
||||||
|
dynamic_deps = depset(),
|
||||||
|
exports = [],
|
||||||
|
linker_input = None,
|
||||||
|
link_once_static_libs = {},
|
||||||
|
)
|
||||||
|
|
||||||
|
# On Windows we need an rc executable
|
||||||
|
if ctx.executable.rc == None:
|
||||||
|
fail("windows_rc: 'rc' tool not provided. Pass rc = '@mongo_windows_toolchain//:rc' (or your wrapper).")
|
||||||
|
|
||||||
|
out = ctx.actions.declare_file(ctx.label.name + ".res")
|
||||||
|
|
||||||
|
defines = []
|
||||||
|
if ctx.attr.windows_version_minimal:
|
||||||
|
ver = ctx.attr.windows_version_minimal[BuildSettingInfo].value
|
||||||
|
defines += _compute_win_defines(ver)
|
||||||
|
defines += ctx.attr.defines
|
||||||
|
|
||||||
|
include_dirs = ["src", ctx.file.src.dirname]
|
||||||
|
|
||||||
|
# Add $(GENDIR)/src (use backslashes for rc.exe friendliness)
|
||||||
|
gen_src = ctx.var["GENDIR"] + "\\src"
|
||||||
|
include_dirs.append(gen_src)
|
||||||
|
|
||||||
|
args = ctx.actions.args()
|
||||||
|
args.add("/nologo")
|
||||||
|
for inc in include_dirs:
|
||||||
|
args.add("/I", inc)
|
||||||
|
for d in defines:
|
||||||
|
args.add("/d", d)
|
||||||
|
args.add("/fo" + out.path) # /fo must be joined
|
||||||
|
args.add(ctx.file.src.path)
|
||||||
|
|
||||||
|
ctx.actions.run(
|
||||||
|
executable = ctx.executable.rc, # your wrapper
|
||||||
|
arguments = [args],
|
||||||
|
inputs = [ctx.file.src] + ctx.files.resources,
|
||||||
|
outputs = [out],
|
||||||
|
mnemonic = "WindowsRC",
|
||||||
|
)
|
||||||
|
|
||||||
|
li = cc_common.create_linker_input(
|
||||||
|
owner = ctx.label,
|
||||||
|
user_link_flags = [out.path], # put .res on the link line
|
||||||
|
additional_inputs = depset([out]), # and declare it as a link input
|
||||||
|
)
|
||||||
|
lc = cc_common.create_linking_context(linker_inputs = depset([li]))
|
||||||
|
new_cc_shared_info = CcSharedLibraryInfo(
|
||||||
|
dynamic_deps = depset(),
|
||||||
|
exports = [],
|
||||||
|
linker_input = li,
|
||||||
|
link_once_static_libs = {},
|
||||||
|
)
|
||||||
|
|
||||||
|
return [DefaultInfo(files = depset([out])), CcInfo(linking_context = lc), new_cc_shared_info]
|
||||||
|
|
||||||
|
windows_rc_rule = rule(
|
||||||
|
implementation = _rc_impl,
|
||||||
|
attrs = {
|
||||||
|
"src": attr.label(mandatory = True, allow_single_file = [".rc"]),
|
||||||
|
"resources": attr.label_list(allow_files = True),
|
||||||
|
"defines": attr.string_list(),
|
||||||
|
"rc": attr.label(executable = True, cfg = "exec", allow_files = True),
|
||||||
|
"windows_version_minimal": attr.label(),
|
||||||
|
"_os_windows": attr.label(
|
||||||
|
default = Label("@platforms//os:windows"),
|
||||||
|
providers = [platform_common.ConstraintValueInfo],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
provides = [CcInfo],
|
||||||
|
fragments = ["cpp"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def windows_rc(name, src, manifest_in = None, icon = None):
|
||||||
|
if manifest_in:
|
||||||
|
# Turn "foo.manifest.in" into "foo.manifest"
|
||||||
|
out_manifest = manifest_in[:-3] if manifest_in.endswith(".in") else manifest_in
|
||||||
|
|
||||||
|
generate_config_header(
|
||||||
|
name = name + "_manifest_gen",
|
||||||
|
checks = "//src/mongo/util:version_constants_gen.py",
|
||||||
|
cpp_defines = [],
|
||||||
|
cpp_linkflags = [],
|
||||||
|
cpp_opts = [],
|
||||||
|
extra_definitions = {
|
||||||
|
"MONGO_DISTMOD": "$(MONGO_DISTMOD)",
|
||||||
|
"MONGO_VERSION": "$(MONGO_VERSION)",
|
||||||
|
"GIT_COMMIT_HASH": "$(GIT_COMMIT_HASH)",
|
||||||
|
} | select({
|
||||||
|
"//bazel/config:js_engine_mozjs": {"js_engine_ver": "mozjs"},
|
||||||
|
"//conditions:default": {"js_engine_ver": "none"},
|
||||||
|
}) | select({
|
||||||
|
"//bazel/config:tcmalloc_google_enabled": {"MONGO_ALLOCATOR": "tcmalloc-google"},
|
||||||
|
"//bazel/config:tcmalloc_gperf_enabled": {"MONGO_ALLOCATOR": "tcmalloc-gperf"},
|
||||||
|
"//conditions:default": {"MONGO_ALLOCATOR": "system"},
|
||||||
|
}) | select({
|
||||||
|
"//bazel/config:build_enterprise_enabled": {"build_enterprise_enabled": "1"},
|
||||||
|
"//conditions:default": {},
|
||||||
|
}),
|
||||||
|
logfile = name + "_manifest_gen.log",
|
||||||
|
output = out_manifest,
|
||||||
|
template = manifest_in,
|
||||||
|
)
|
||||||
|
|
||||||
|
resources = ["//src/mongo/util:rc_constants_gen"]
|
||||||
|
if manifest_in:
|
||||||
|
resources.append(name + "_manifest_gen")
|
||||||
|
if icon:
|
||||||
|
resources.append(icon)
|
||||||
|
|
||||||
|
windows_rc_rule(
|
||||||
|
name = name,
|
||||||
|
src = src,
|
||||||
|
resources = resources,
|
||||||
|
rc = select({
|
||||||
|
"@platforms//os:windows": "@mongo_windows_toolchain//:rc",
|
||||||
|
"//conditions:default": None,
|
||||||
|
}),
|
||||||
|
windows_version_minimal = "//bazel/config:win_min_version",
|
||||||
|
)
|
||||||
|
|||||||
@ -39,6 +39,14 @@ mongo_windows_cc_toolchain_config(
|
|||||||
supports_parse_showincludes = True,
|
supports_parse_showincludes = True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
exports_files(["rc_wrapper.cmd"])
|
||||||
|
|
||||||
|
alias(
|
||||||
|
name = "rc",
|
||||||
|
actual = "rc_wrapper.cmd",
|
||||||
|
visibility = ["//visibility:public"]
|
||||||
|
)
|
||||||
|
|
||||||
cc_toolchain(
|
cc_toolchain(
|
||||||
name = "mongo_win_cc_toolchain",
|
name = "mongo_win_cc_toolchain",
|
||||||
all_files = ":all_files",
|
all_files = ":all_files",
|
||||||
|
|||||||
@ -11,6 +11,41 @@ load(
|
|||||||
"setup_vc_env_vars",
|
"setup_vc_env_vars",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _compute_rc_path(ctx, env_vars):
|
||||||
|
# Allow an explicit override from CI/dev
|
||||||
|
rc_path = ctx.os.environ.get("RC_PATH")
|
||||||
|
if rc_path and ctx.path(rc_path).exists:
|
||||||
|
return rc_path.replace("\\", "/")
|
||||||
|
|
||||||
|
winsdkdir = (env_vars.get("WINDOWSSDKDIR") or "").rstrip("\\/")
|
||||||
|
sdkver = (env_vars.get("WindowsSDKVersion") or "").strip("\\/")
|
||||||
|
|
||||||
|
# Typical locations in recent SDKs
|
||||||
|
candidates = []
|
||||||
|
if winsdkdir:
|
||||||
|
if sdkver:
|
||||||
|
candidates += [
|
||||||
|
"%s\\bin\\%s\\x64\\rc.exe" % (winsdkdir, sdkver),
|
||||||
|
"%s\\bin\\%s\\x86\\rc.exe" % (winsdkdir, sdkver),
|
||||||
|
]
|
||||||
|
candidates += [
|
||||||
|
"%s\\bin\\x64\\rc.exe" % winsdkdir,
|
||||||
|
"%s\\bin\\x86\\rc.exe" % winsdkdir,
|
||||||
|
]
|
||||||
|
|
||||||
|
# Fallback to LLVM if present
|
||||||
|
llvm = ctx.os.environ.get("BAZEL_LLVM")
|
||||||
|
if llvm:
|
||||||
|
candidates += [
|
||||||
|
"%s\\bin\\llvm-rc.exe" % llvm,
|
||||||
|
"%s\\bin\\rc.exe" % llvm, # some layouts
|
||||||
|
]
|
||||||
|
|
||||||
|
for c in candidates:
|
||||||
|
if ctx.path(c).exists:
|
||||||
|
return c.replace("\\", "/")
|
||||||
|
return None
|
||||||
|
|
||||||
def _impl_gen_windows_toolchain_build_file(ctx):
|
def _impl_gen_windows_toolchain_build_file(ctx):
|
||||||
if "windows" not in ctx.os.name:
|
if "windows" not in ctx.os.name:
|
||||||
ctx.file(
|
ctx.file(
|
||||||
@ -85,6 +120,21 @@ def _impl_gen_windows_toolchain_build_file(ctx):
|
|||||||
# purpose.
|
# purpose.
|
||||||
ctx.file("windows_toolchain_config.json", json.encode(substitutions), executable = False)
|
ctx.file("windows_toolchain_config.json", json.encode(substitutions), executable = False)
|
||||||
|
|
||||||
|
# --- expose rc.exe/llvm-rc.exe as a stable label in this repo ---
|
||||||
|
# Ask VS env for SDK location/version (setup_vc_env_vars already did that).
|
||||||
|
# Add WindowsSDKVersion to your setup_vc_env_vars() if not already included.
|
||||||
|
rc_path = _compute_rc_path(ctx, vars)
|
||||||
|
if not rc_path:
|
||||||
|
auto_configure_fail(
|
||||||
|
"Could not locate rc.exe (or llvm-rc.exe). Set RC_PATH or ensure Windows SDK is installed and WindowsSDKVersion is exported.",
|
||||||
|
)
|
||||||
|
|
||||||
|
ctx.file(
|
||||||
|
"rc_wrapper.cmd",
|
||||||
|
"@echo off\r\nsetlocal\r\nset \"INCLUDE=%s\"\r\nset \"PATH=%s\"\r\n\"%s\" %%*\r\n" % (vars["INCLUDE"], vars["PATH"], rc_path),
|
||||||
|
executable = True,
|
||||||
|
)
|
||||||
|
|
||||||
ctx.report_progress("Generating toolchain build file")
|
ctx.report_progress("Generating toolchain build file")
|
||||||
ctx.template(
|
ctx.template(
|
||||||
"BUILD.bazel",
|
"BUILD.bazel",
|
||||||
|
|||||||
@ -36,6 +36,16 @@ load(
|
|||||||
"with_feature_set",
|
"with_feature_set",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The values are populated from the following link:
|
||||||
|
# https://learn.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
|
||||||
|
# For future versions, please add it in the map.
|
||||||
|
MIN_VER_MAP = {
|
||||||
|
"10": {
|
||||||
|
"win": "0x0A00",
|
||||||
|
"ddi": "0x0A000000",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
all_compile_actions = [
|
all_compile_actions = [
|
||||||
ACTION_NAMES.c_compile,
|
ACTION_NAMES.c_compile,
|
||||||
ACTION_NAMES.cpp_compile,
|
ACTION_NAMES.cpp_compile,
|
||||||
@ -85,25 +95,15 @@ all_link_actions = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
def get_windows_mimimun_version_feature(ctx):
|
def get_windows_mimimun_version_feature(ctx):
|
||||||
# The values are populated from the following link:
|
|
||||||
# https://learn.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
|
|
||||||
# For future versions, please add it in the map.
|
|
||||||
min_ver_map = {
|
|
||||||
"10": {
|
|
||||||
"win": "0x0A00",
|
|
||||||
"ddi": "0x0A000000",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if BuildSettingInfo not in ctx.attr.windows_version_minimal:
|
if BuildSettingInfo not in ctx.attr.windows_version_minimal:
|
||||||
fail("windows_version_minimal attribute value is not a build flag.")
|
fail("windows_version_minimal attribute value is not a build flag.")
|
||||||
|
|
||||||
ver = ctx.attr.windows_version_minimal[BuildSettingInfo].value
|
ver = ctx.attr.windows_version_minimal[BuildSettingInfo].value
|
||||||
if ver not in min_ver_map:
|
if ver not in MIN_VER_MAP:
|
||||||
error_msg = "Windows mininum version {} does not exist. These are the minimum versions that are supported: {}".format(ver, min_ver_map.keys())
|
error_msg = "Windows mininum version {} does not exist. These are the minimum versions that are supported: {}".format(ver, MIN_VER_MAP.keys())
|
||||||
fail(error_msg)
|
fail(error_msg)
|
||||||
|
|
||||||
min_ver = min_ver_map[ver]
|
min_ver = MIN_VER_MAP[ver]
|
||||||
return feature(
|
return feature(
|
||||||
name = "windows_version_minimum",
|
name = "windows_version_minimum",
|
||||||
enabled = True,
|
enabled = True,
|
||||||
|
|||||||
@ -351,7 +351,7 @@ def setup_vc_env_vars(repository_ctx, vc_path, envvars = [], allow_empty = False
|
|||||||
dictionary of the envvars
|
dictionary of the envvars
|
||||||
"""
|
"""
|
||||||
if not envvars:
|
if not envvars:
|
||||||
envvars = ["PATH", "INCLUDE", "LIB", "WINDOWSSDKDIR"]
|
envvars = ["PATH", "INCLUDE", "LIB", "WINDOWSSDKDIR", "WindowsSDKVersion"]
|
||||||
|
|
||||||
vcvars_script = _find_vcvars_bat_script(repository_ctx, vc_path)
|
vcvars_script = _find_vcvars_bat_script(repository_ctx, vc_path)
|
||||||
if not vcvars_script:
|
if not vcvars_script:
|
||||||
|
|||||||
@ -1,6 +1,27 @@
|
|||||||
# General starlark utility functions
|
# General starlark utility functions
|
||||||
load("//bazel/platforms:normalize.bzl", "ARCH_NORMALIZE_MAP")
|
load("//bazel/platforms:normalize.bzl", "ARCH_NORMALIZE_MAP")
|
||||||
|
|
||||||
|
def write_target_impl(ctx):
|
||||||
|
out = ctx.actions.declare_file(ctx.label.name + ".gen_source_list")
|
||||||
|
ctx.actions.write(
|
||||||
|
out,
|
||||||
|
"//" + ctx.label.package + ":" + ctx.attr.target_name,
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
DefaultInfo(
|
||||||
|
files = depset([out]),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
write_target = rule(
|
||||||
|
write_target_impl,
|
||||||
|
attrs = {
|
||||||
|
"target_name": attr.string(
|
||||||
|
doc = "the name of the target to record",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
def retry_download_and_extract(ctx, tries, **kwargs):
|
def retry_download_and_extract(ctx, tries, **kwargs):
|
||||||
sleep_time = 1
|
sleep_time = 1
|
||||||
for attempt in range(tries):
|
for attempt in range(tries):
|
||||||
|
|||||||
@ -10,6 +10,7 @@ py_library(
|
|||||||
"install_modules.py",
|
"install_modules.py",
|
||||||
"lint.py",
|
"lint.py",
|
||||||
"plus_interface.py",
|
"plus_interface.py",
|
||||||
|
"set_mongo_variables.py",
|
||||||
"wrapper_debug.py",
|
"wrapper_debug.py",
|
||||||
"wrapper_hook.py",
|
"wrapper_hook.py",
|
||||||
],
|
],
|
||||||
|
|||||||
@ -11,6 +11,8 @@ import sys
|
|||||||
REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
|
REPO_ROOT = pathlib.Path(__file__).parent.parent.parent
|
||||||
sys.path.append(str(REPO_ROOT))
|
sys.path.append(str(REPO_ROOT))
|
||||||
|
|
||||||
|
from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc
|
||||||
|
|
||||||
|
|
||||||
def run_pty_command(cmd):
|
def run_pty_command(cmd):
|
||||||
stdout = None
|
stdout = None
|
||||||
@ -43,6 +45,8 @@ def run_pty_command(cmd):
|
|||||||
|
|
||||||
|
|
||||||
def generate_compiledb(bazel_bin, persistent_compdb, enterprise):
|
def generate_compiledb(bazel_bin, persistent_compdb, enterprise):
|
||||||
|
# compiledb ignores command line args so just make a version rc file in anycase
|
||||||
|
write_mongo_variables_bazelrc([])
|
||||||
if persistent_compdb:
|
if persistent_compdb:
|
||||||
info_proc = subprocess.run(
|
info_proc = subprocess.run(
|
||||||
[bazel_bin, "info", "output_base"], capture_output=True, text=True
|
[bazel_bin, "info", "output_base"], capture_output=True, text=True
|
||||||
|
|||||||
@ -22,6 +22,7 @@ def write_workstation_bazelrc(args):
|
|||||||
commit = "Unknown"
|
commit = "Unknown"
|
||||||
user = "Unknown"
|
user = "Unknown"
|
||||||
hostname = "Unknown"
|
hostname = "Unknown"
|
||||||
|
base_branch = "Unknown"
|
||||||
try:
|
try:
|
||||||
repo = git.Repo()
|
repo = git.Repo()
|
||||||
except Exception:
|
except Exception:
|
||||||
@ -63,6 +64,10 @@ def write_workstation_bazelrc(args):
|
|||||||
|
|
||||||
if os.environ.get("CI") is not None:
|
if os.environ.get("CI") is not None:
|
||||||
user = os.environ.get("author_email", "Unknown")
|
user = os.environ.get("author_email", "Unknown")
|
||||||
|
# This is the branch that the PR is merging into
|
||||||
|
base_branch = os.environ.get("github_pr_base_branch", "Unknown")
|
||||||
|
# Replace the branch with the head branch if this is a PR check / merge queue run
|
||||||
|
branch = os.environ.get("github_pr_head_branch", branch)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
hostname = socket.gethostname()
|
hostname = socket.gethostname()
|
||||||
@ -71,7 +76,7 @@ def write_workstation_bazelrc(args):
|
|||||||
|
|
||||||
filtered_args = args[1:]
|
filtered_args = args[1:]
|
||||||
if "--" in filtered_args:
|
if "--" in filtered_args:
|
||||||
filtered_args = filtered_args[:filtered_args.index("--")] + ["--", "(REDACTED)"]
|
filtered_args = filtered_args[: filtered_args.index("--")] + ["--", "(REDACTED)"]
|
||||||
|
|
||||||
developer_build = os.environ.get("CI") is None
|
developer_build = os.environ.get("CI") is None
|
||||||
filtered_command_line = " ".join(filtered_args)
|
filtered_command_line = " ".join(filtered_args)
|
||||||
@ -87,6 +92,7 @@ common --bes_keywords=engflow:BuildScmBranch={branch}
|
|||||||
common --bes_keywords=engflow:BuildScmRevision={commit}
|
common --bes_keywords=engflow:BuildScmRevision={commit}
|
||||||
common --bes_keywords=engflow:BuildScmStatus={status}
|
common --bes_keywords=engflow:BuildScmStatus={status}
|
||||||
common --bes_keywords=rawCommandLineBase64={b64_cmd_line}
|
common --bes_keywords=rawCommandLineBase64={b64_cmd_line}
|
||||||
|
common --bes_keywords=base_branch={base_branch}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if developer_build:
|
if developer_build:
|
||||||
|
|||||||
37
bazel/wrapper_hook/remove_auto_header_dirs.py
Normal file
37
bazel/wrapper_hook/remove_auto_header_dirs.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import shutil
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
TARGET = ".auto_header"
|
||||||
|
|
||||||
|
|
||||||
|
def delete_dir(path: str):
|
||||||
|
# Don't follow/delete symlinked dirs
|
||||||
|
if os.path.islink(path):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def find_targets(root: str):
|
||||||
|
for dirpath, dirnames, _ in os.walk(root):
|
||||||
|
# If TARGET exists directly under this directory
|
||||||
|
if TARGET in dirnames:
|
||||||
|
target_path = os.path.join(dirpath, TARGET)
|
||||||
|
yield target_path
|
||||||
|
# Prevent os.walk from descending into it
|
||||||
|
dirnames.remove(TARGET)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_up_auto_header_dirs(ROOT):
|
||||||
|
targets = list(find_targets(ROOT))
|
||||||
|
if not targets:
|
||||||
|
return
|
||||||
|
|
||||||
|
workers = min(32, (os.cpu_count() or 8) * 2)
|
||||||
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||||
|
_ = list(ex.map(delete_dir, targets))
|
||||||
48
bazel/wrapper_hook/set_mongo_variables.py
Normal file
48
bazel/wrapper_hook/set_mongo_variables.py
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import platform
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
ARCH_NORMALIZE_MAP = {
|
||||||
|
"amd64": "x86_64",
|
||||||
|
"x86_64": "x86_64",
|
||||||
|
"arm64": "aarch64",
|
||||||
|
"aarch64": "aarch64",
|
||||||
|
"ppc64le": "ppc64le",
|
||||||
|
"s390x": "s390x",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_mongo_arch(args):
|
||||||
|
arch = platform.machine().lower()
|
||||||
|
if arch in ARCH_NORMALIZE_MAP:
|
||||||
|
return ARCH_NORMALIZE_MAP[arch]
|
||||||
|
else:
|
||||||
|
return arch
|
||||||
|
|
||||||
|
|
||||||
|
def get_mongo_version(args):
|
||||||
|
proc = subprocess.run(["git", "describe", "--abbrev=0"], capture_output=True, text=True)
|
||||||
|
return proc.stdout.strip()[1:]
|
||||||
|
|
||||||
|
|
||||||
|
def write_mongo_variables_bazelrc(args):
|
||||||
|
mongo_version = get_mongo_version(args)
|
||||||
|
mongo_arch = get_mongo_arch(args)
|
||||||
|
|
||||||
|
repo_root = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent
|
||||||
|
version_file = os.path.join(repo_root, ".bazelrc.mongo_variables")
|
||||||
|
existing_hash = ""
|
||||||
|
if os.path.exists(version_file):
|
||||||
|
with open(version_file, encoding="utf-8") as f:
|
||||||
|
existing_hash = hashlib.md5(f.read().encode()).hexdigest()
|
||||||
|
|
||||||
|
bazelrc_contents = f"""
|
||||||
|
common --define=MONGO_ARCH={mongo_arch}
|
||||||
|
common --define=MONGO_VERSION={mongo_version}
|
||||||
|
"""
|
||||||
|
current_hash = hashlib.md5(bazelrc_contents.encode()).hexdigest()
|
||||||
|
if existing_hash != current_hash:
|
||||||
|
with open(version_file, "w", encoding="utf-8") as f:
|
||||||
|
f.write(bazelrc_contents)
|
||||||
@ -22,6 +22,10 @@ def main():
|
|||||||
from bazel.wrapper_hook.engflow_check import engflow_auth
|
from bazel.wrapper_hook.engflow_check import engflow_auth
|
||||||
from bazel.wrapper_hook.generate_common_bes_bazelrc import write_workstation_bazelrc
|
from bazel.wrapper_hook.generate_common_bes_bazelrc import write_workstation_bazelrc
|
||||||
from bazel.wrapper_hook.plus_interface import check_bazel_command_type, test_runner_interface
|
from bazel.wrapper_hook.plus_interface import check_bazel_command_type, test_runner_interface
|
||||||
|
from bazel.wrapper_hook.remove_auto_header_dirs import clean_up_auto_header_dirs
|
||||||
|
from bazel.wrapper_hook.set_mongo_variables import write_mongo_variables_bazelrc
|
||||||
|
|
||||||
|
clean_up_auto_header_dirs(REPO_ROOT)
|
||||||
|
|
||||||
# This is used to autogenerate a BUILD.bazel that creates
|
# This is used to autogenerate a BUILD.bazel that creates
|
||||||
# Filegroups for select tags - used to group targets for installing
|
# Filegroups for select tags - used to group targets for installing
|
||||||
@ -38,10 +42,15 @@ def main():
|
|||||||
)
|
)
|
||||||
args += ["--build_enterprise=False", "--config=local"]
|
args += ["--build_enterprise=False", "--config=local"]
|
||||||
|
|
||||||
|
if any(arg.startswith("--include_mongot") for arg in args):
|
||||||
|
os.makedirs("mongot-localdev", exist_ok=True)
|
||||||
|
|
||||||
engflow_auth(args)
|
engflow_auth(args)
|
||||||
|
|
||||||
write_workstation_bazelrc(args)
|
write_workstation_bazelrc(args)
|
||||||
|
|
||||||
|
write_mongo_variables_bazelrc(args)
|
||||||
|
|
||||||
args = test_runner_interface(
|
args = test_runner_interface(
|
||||||
sys.argv[1:],
|
sys.argv[1:],
|
||||||
autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1",
|
autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1",
|
||||||
|
|||||||
@ -150,6 +150,10 @@ py_binary(
|
|||||||
"jsonschema",
|
"jsonschema",
|
||||||
group = "build-metrics",
|
group = "build-metrics",
|
||||||
),
|
),
|
||||||
|
dependency(
|
||||||
|
"license-expression",
|
||||||
|
group = "lint",
|
||||||
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -7,8 +7,15 @@ import sys
|
|||||||
import click
|
import click
|
||||||
import structlog
|
import structlog
|
||||||
from pydantic.main import BaseModel
|
from pydantic.main import BaseModel
|
||||||
|
from urllib3.util import Retry
|
||||||
|
|
||||||
from evergreen.api import EvergreenApi, RetryingEvergreenApi
|
from evergreen.api import (
|
||||||
|
DEFAULT_HTTP_RETRY_ATTEMPTS,
|
||||||
|
DEFAULT_HTTP_RETRY_BACKOFF_FACTOR,
|
||||||
|
DEFAULT_HTTP_RETRY_CODES,
|
||||||
|
EvergreenApi,
|
||||||
|
RetryingEvergreenApi,
|
||||||
|
)
|
||||||
|
|
||||||
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
||||||
if __name__ == "__main__" and __package__ is None:
|
if __name__ == "__main__" and __package__ is None:
|
||||||
@ -137,6 +144,14 @@ def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None:
|
|||||||
enable_logging(verbose)
|
enable_logging(verbose)
|
||||||
expansions = EvgExpansions.from_yaml_file(expansion_file)
|
expansions = EvgExpansions.from_yaml_file(expansion_file)
|
||||||
evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config, log_on_error=True)
|
evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config, log_on_error=True)
|
||||||
|
evg_api._http_retry = Retry(
|
||||||
|
# this is a way to reuse all of Evergreen's logic, but to bump up the number of attempts
|
||||||
|
total=DEFAULT_HTTP_RETRY_ATTEMPTS + 10,
|
||||||
|
backoff_factor=DEFAULT_HTTP_RETRY_BACKOFF_FACTOR,
|
||||||
|
status_forcelist=DEFAULT_HTTP_RETRY_CODES,
|
||||||
|
raise_on_status=False,
|
||||||
|
raise_on_redirect=False,
|
||||||
|
)
|
||||||
|
|
||||||
activate_task(expansions, evg_api)
|
activate_task(expansions, evg_api)
|
||||||
|
|
||||||
|
|||||||
@ -6,6 +6,7 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
import sys
|
import sys
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
@ -115,7 +116,7 @@ class TimeoutOverrides(BaseModel):
|
|||||||
overrides = [
|
overrides = [
|
||||||
override
|
override
|
||||||
for override in self.overrides.get(build_variant, [])
|
for override in self.overrides.get(build_variant, [])
|
||||||
if override.task == task_name
|
if re.search(override.task, task_name)
|
||||||
]
|
]
|
||||||
if overrides:
|
if overrides:
|
||||||
if len(overrides) > 1:
|
if len(overrides) > 1:
|
||||||
|
|||||||
@ -61,8 +61,8 @@ def generate_version_expansions():
|
|||||||
raise ValueError("Unable to parse version from stdin and no version.json provided")
|
raise ValueError("Unable to parse version from stdin and no version.json provided")
|
||||||
|
|
||||||
if version_parts[0]:
|
if version_parts[0]:
|
||||||
expansions["suffix"] = "latest"
|
expansions["suffix"] = "v8.2-latest"
|
||||||
expansions["src_suffix"] = "latest"
|
expansions["src_suffix"] = "v8.2-latest"
|
||||||
expansions["is_release"] = "false"
|
expansions["is_release"] = "false"
|
||||||
else:
|
else:
|
||||||
expansions["suffix"] = version_line
|
expansions["suffix"] = version_line
|
||||||
|
|||||||
@ -31,10 +31,11 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
from subprocess import check_output
|
from subprocess import CalledProcessError, check_output
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from packaging.version import Version
|
from packaging.version import Version
|
||||||
|
from retry import retry
|
||||||
|
|
||||||
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
||||||
if __name__ == "__main__" and __package__ is None:
|
if __name__ == "__main__" and __package__ is None:
|
||||||
@ -104,6 +105,11 @@ def get_tags() -> List[str]:
|
|||||||
return list(gen_tags())
|
return list(gen_tags())
|
||||||
|
|
||||||
|
|
||||||
|
@retry(tries=3, delay=5)
|
||||||
|
def _show_with_retry(tag: str, path: str):
|
||||||
|
return check_output(["git", "show", f"{tag}:{path}"])
|
||||||
|
|
||||||
|
|
||||||
def make_idl_directories(tags: List[str], destination: str) -> None:
|
def make_idl_directories(tags: List[str], destination: str) -> None:
|
||||||
"""For each tag, construct a source tree containing only its IDL files."""
|
"""For each tag, construct a source tree containing only its IDL files."""
|
||||||
LOGGER.info("Clearing destination directory '%s'", destination)
|
LOGGER.info("Clearing destination directory '%s'", destination)
|
||||||
@ -116,7 +122,12 @@ def make_idl_directories(tags: List[str], destination: str) -> None:
|
|||||||
if not path.endswith(".idl"):
|
if not path.endswith(".idl"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
contents = check_output(["git", "show", f"{tag}:{path}"]).decode()
|
try:
|
||||||
|
contents = _show_with_retry(tag, path).decode()
|
||||||
|
except CalledProcessError as e:
|
||||||
|
LOGGER.error("Failed to run git show command after multiple retries: %s", str(e))
|
||||||
|
raise e
|
||||||
|
|
||||||
output_path = os.path.join(directory, path)
|
output_path = os.path.join(directory, path)
|
||||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||||
with open(output_path, "w+") as fd:
|
with open(output_path, "w+") as fd:
|
||||||
|
|||||||
@ -71,10 +71,6 @@ def install_bazel(binary_directory: str) -> str:
|
|||||||
is_bazelisk_supported = normalized_arch not in ["ppc64le", "s390x"]
|
is_bazelisk_supported = normalized_arch not in ["ppc64le", "s390x"]
|
||||||
binary_filename = "bazelisk"
|
binary_filename = "bazelisk"
|
||||||
binary_path = os.path.join(binary_directory, binary_filename)
|
binary_path = os.path.join(binary_directory, binary_filename)
|
||||||
if os.path.exists(binary_path):
|
|
||||||
print(f"{binary_filename} already exists ({binary_path}), skipping download")
|
|
||||||
_set_bazel_permissions(binary_path)
|
|
||||||
return binary_path
|
|
||||||
|
|
||||||
if is_bazelisk_supported:
|
if is_bazelisk_supported:
|
||||||
print(f"Downloading {binary_filename}...")
|
print(f"Downloading {binary_filename}...")
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
# This script validates the msi can be installed, uninstalled, and checks for the default install location and some of the possible install files
|
# This script validates the msi can be installed, uninstalled, and checks for the default install location and some of the possible install files
|
||||||
import glob
|
import glob
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
@ -87,6 +88,8 @@ def validate_files(is_enterprise):
|
|||||||
print(f"File exists: {file_match[0]}")
|
print(f"File exists: {file_match[0]}")
|
||||||
if file_match[0].endswith(".exe"):
|
if file_match[0].endswith(".exe"):
|
||||||
validate_help(file_match[0])
|
validate_help(file_match[0])
|
||||||
|
if file_match[0].endswith("mongod.exe"):
|
||||||
|
validate_version(file_match[0])
|
||||||
else:
|
else:
|
||||||
print(f"Error: {file_path} could not be found.")
|
print(f"Error: {file_path} could not be found.")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@ -103,6 +106,26 @@ def validate_help(exe_path):
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
# Make sure we have a proper git version in the windows release
|
||||||
|
def validate_version(exe_path):
|
||||||
|
try:
|
||||||
|
version_command = [exe_path, "--version"]
|
||||||
|
print(f"Calling '{exe_path}' with command: {' '.join(version_command)}...")
|
||||||
|
result = subprocess.run(version_command, check=True, stdout=subprocess.PIPE, text=True)
|
||||||
|
print(f"{exe_path} called version successfully.")
|
||||||
|
match = re.search('.*"gitVersion": "[0-9a-fA-F]{40}".*', result.stdout)
|
||||||
|
if match:
|
||||||
|
print("Found a valid git version.")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
print("--version command did not contain a valid git version in gitVersion. Stdout:")
|
||||||
|
print(result.stdout)
|
||||||
|
sys.exit(1)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Error while calling version for {exe_path}: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("Usage: python msi_validation.py <path_to_msi>")
|
print("Usage: python msi_validation.py <path_to_msi>")
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import uuid
|
|||||||
from concurrent import futures
|
from concurrent import futures
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Generator, List, Optional, Set, Tuple
|
from typing import Any, Dict, Generator, List, Optional, Set, Tuple
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import docker
|
import docker
|
||||||
import docker.errors
|
import docker.errors
|
||||||
@ -24,6 +25,8 @@ from docker.models.images import Image
|
|||||||
from retry.api import retry_call
|
from retry.api import retry_call
|
||||||
from simple_report import Report, Result
|
from simple_report import Report, Result
|
||||||
|
|
||||||
|
from buildscripts.resmokelib.utils import evergreen_conn
|
||||||
|
|
||||||
root = logging.getLogger()
|
root = logging.getLogger()
|
||||||
root.setLevel(logging.DEBUG)
|
root.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
@ -33,6 +36,66 @@ formatter = logging.Formatter("[%(asctime)s]%(levelname)s:%(message)s")
|
|||||||
handler.setFormatter(formatter)
|
handler.setFormatter(formatter)
|
||||||
root.addHandler(handler)
|
root.addHandler(handler)
|
||||||
|
|
||||||
|
|
||||||
|
def download_packages_from_build(build_id: str, download_dir: Path) -> Path:
|
||||||
|
"""
|
||||||
|
Download the packages artifact from the Evergreen API.
|
||||||
|
|
||||||
|
This is needed because private artifacts require authenticated access.
|
||||||
|
The Evergreen API client handles authentication and downloads the file
|
||||||
|
to a local path that can be mounted into Docker containers.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
build_id: The Evergreen build ID to search for the package task
|
||||||
|
download_dir: Directory where the packages tarball should be downloaded
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The local path to the downloaded packages tarball
|
||||||
|
"""
|
||||||
|
logging.info("Fetching packages artifact from Evergreen API for build: %s", build_id)
|
||||||
|
|
||||||
|
evg_api = evergreen_conn.get_evergreen_api()
|
||||||
|
tasks = evg_api.tasks_by_build(build_id)
|
||||||
|
|
||||||
|
package_task = None
|
||||||
|
for task in tasks:
|
||||||
|
if task.display_name == "package":
|
||||||
|
package_task = task
|
||||||
|
break
|
||||||
|
|
||||||
|
if package_task is None:
|
||||||
|
raise RuntimeError(f"Could not find 'package' task in build {build_id}")
|
||||||
|
|
||||||
|
logging.info("Found package task: %s", package_task.task_id)
|
||||||
|
|
||||||
|
packages_url = None
|
||||||
|
for artifact in package_task.artifacts:
|
||||||
|
if artifact.name == "Packages":
|
||||||
|
packages_url = artifact.url
|
||||||
|
logging.info("Found Packages artifact URL: %s", packages_url)
|
||||||
|
break
|
||||||
|
|
||||||
|
if packages_url is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Could not find 'Packages' artifact for package task {package_task.task_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Download the packages file
|
||||||
|
download_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
local_path = download_dir / "packages.tgz"
|
||||||
|
|
||||||
|
logging.info("Downloading packages to: %s", local_path)
|
||||||
|
response = requests.get(packages_url, stream=True, timeout=300)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
with open(local_path, "wb") as f:
|
||||||
|
for chunk in response.iter_content(chunk_size=8192):
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
|
logging.info("Downloaded packages successfully: %s", local_path)
|
||||||
|
return local_path
|
||||||
|
|
||||||
|
|
||||||
PACKAGE_MANAGER_COMMANDS = {
|
PACKAGE_MANAGER_COMMANDS = {
|
||||||
"apt": {
|
"apt": {
|
||||||
"update": "export DEBIAN_FRONTEND=noninteractive && apt-get update -y",
|
"update": "export DEBIAN_FRONTEND=noninteractive && apt-get update -y",
|
||||||
@ -249,6 +312,16 @@ VERSIONS_TO_SKIP: Set[str] = set(
|
|||||||
)
|
)
|
||||||
DISABLED_TESTS: Set[Tuple[str, str]] = set()
|
DISABLED_TESTS: Set[Tuple[str, str]] = set()
|
||||||
|
|
||||||
|
VALID_TAR_DIRECTORY_ARCHITECTURES = [
|
||||||
|
"linux-aarch64",
|
||||||
|
"linux-x86_64",
|
||||||
|
"linux-ppc64le",
|
||||||
|
"linux-s390x",
|
||||||
|
"macos-aarch64",
|
||||||
|
"macos-x86_64",
|
||||||
|
"windows-x86_64",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
@dataclasses.dataclass
|
||||||
class Test:
|
class Test:
|
||||||
@ -350,7 +423,7 @@ def run_test(test: Test, client: DockerClient) -> Result:
|
|||||||
"yum -y install yum-utils epel-release",
|
"yum -y install yum-utils epel-release",
|
||||||
"yum-config-manager --enable epel",
|
"yum-config-manager --enable epel",
|
||||||
]
|
]
|
||||||
if test.os_name.startswith("debian92"):
|
if test.os_name.startswith("debian92") or test.os_name.startswith("debian10"):
|
||||||
# Adapted from https://stackoverflow.com/questions/76094428/debian-stretch-repositories-404-not-found
|
# Adapted from https://stackoverflow.com/questions/76094428/debian-stretch-repositories-404-not-found
|
||||||
# Debian92 renamed its repos to archive
|
# Debian92 renamed its repos to archive
|
||||||
# The first two sed commands are to replace debian92's sources list to archive repo
|
# The first two sed commands are to replace debian92's sources list to archive repo
|
||||||
@ -370,8 +443,28 @@ def run_test(test: Test, client: DockerClient) -> Result:
|
|||||||
commands.append(f"ln -s {test.python_command} /usr/bin/python3")
|
commands.append(f"ln -s {test.python_command} /usr/bin/python3")
|
||||||
|
|
||||||
os.makedirs(log_external_path.parent, exist_ok=True)
|
os.makedirs(log_external_path.parent, exist_ok=True)
|
||||||
|
|
||||||
|
# Transform file:// URLs to use Docker-mounted paths
|
||||||
|
docker_packages_urls = []
|
||||||
|
for url in test.packages_urls:
|
||||||
|
parsed_url = urlparse(url)
|
||||||
|
if parsed_url.scheme == "file":
|
||||||
|
host_path = Path(parsed_url.path)
|
||||||
|
# Check if the path is within the test_external_root (buildscripts dir)
|
||||||
|
try:
|
||||||
|
relative_path = host_path.relative_to(test_external_root)
|
||||||
|
# Transform to Docker-mounted path
|
||||||
|
docker_path = Path(test_docker_root) / relative_path
|
||||||
|
docker_packages_urls.append(f"file://{docker_path}")
|
||||||
|
except ValueError:
|
||||||
|
# Path is not relative to test_external_root
|
||||||
|
docker_packages_urls.append(url)
|
||||||
|
else:
|
||||||
|
docker_packages_urls.append(url)
|
||||||
|
|
||||||
commands.append(
|
commands.append(
|
||||||
f"python3 /mnt/package_test/package_test_internal.py {log_docker_path} {' '.join(test.packages_urls)}"
|
f"python3 /mnt/package_test/package_test_internal.py {log_docker_path} "
|
||||||
|
f"{' '.join(docker_packages_urls)}"
|
||||||
)
|
)
|
||||||
logging.debug(
|
logging.debug(
|
||||||
"Attempting to run the following docker commands:\n\t%s",
|
"Attempting to run the following docker commands:\n\t%s",
|
||||||
@ -524,6 +617,16 @@ def get_edition_alias(edition_name: str) -> str:
|
|||||||
return edition_name
|
return edition_name
|
||||||
|
|
||||||
|
|
||||||
|
def validate_top_level_directory(tar_name: str):
|
||||||
|
command = f"tar -tf {tar_name} | head -n 1 | awk -F/ '{{print $1}}'"
|
||||||
|
proc = subprocess.run(command, capture_output=True, shell=True, text=True)
|
||||||
|
top_level_directory = proc.stdout.strip()
|
||||||
|
if all(os_arch not in top_level_directory for os_arch in VALID_TAR_DIRECTORY_ARCHITECTURES):
|
||||||
|
raise Exception(
|
||||||
|
f"Found an unexpected os-arch pairing as the top level directory. Top level directory: {top_level_directory}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
arches: Set[str] = set()
|
arches: Set[str] = set()
|
||||||
oses: Set[str] = set()
|
oses: Set[str] = set()
|
||||||
editions: Set[str] = set()
|
editions: Set[str] = set()
|
||||||
@ -600,6 +703,13 @@ branch_test_parser.add_argument(
|
|||||||
branch_test_parser.add_argument(
|
branch_test_parser.add_argument(
|
||||||
"-v", "--server-version", type=str, help="Server version being tested", required=True
|
"-v", "--server-version", type=str, help="Server version being tested", required=True
|
||||||
)
|
)
|
||||||
|
branch_test_parser.add_argument(
|
||||||
|
"--evg-build-id",
|
||||||
|
type=str,
|
||||||
|
help="Evergreen build ID. If provided, the packages URL will be fetched "
|
||||||
|
"from the Evergreen API (required for private artifacts).",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.command == "release":
|
if args.command == "release":
|
||||||
@ -624,9 +734,22 @@ tests: List[Test] = []
|
|||||||
urls: List[str] = []
|
urls: List[str] = []
|
||||||
|
|
||||||
if args.command == "branch":
|
if args.command == "branch":
|
||||||
|
# If evg-build-id is provided, download the packages locally using the Evergreen API
|
||||||
|
# This is required for private artifacts which need authenticated access
|
||||||
|
local_packages_path: Optional[Path] = None
|
||||||
|
if args.evg_build_id:
|
||||||
|
download_dir = Path(__file__).parent / "downloaded_packages"
|
||||||
|
local_packages_path = download_packages_from_build(args.evg_build_id, download_dir)
|
||||||
|
|
||||||
for test_pair in args.test:
|
for test_pair in args.test:
|
||||||
test_os = test_pair[0]
|
test_os = test_pair[0]
|
||||||
urls = [test_pair[1]]
|
# Use local path if packages were downloaded, otherwise use the URL from --test argument
|
||||||
|
if local_packages_path:
|
||||||
|
# Use file:// protocol for local files - package_test_internal.py will handle this
|
||||||
|
package_url = f"file://{local_packages_path.resolve()}"
|
||||||
|
else:
|
||||||
|
package_url = test_pair[1]
|
||||||
|
urls = [package_url]
|
||||||
if test_os not in OS_DOCKER_LOOKUP:
|
if test_os not in OS_DOCKER_LOOKUP:
|
||||||
logging.error(
|
logging.error(
|
||||||
"We have not seen this OS %s before, please add it to OS_DOCKER_LOOKUP", test_os
|
"We have not seen this OS %s before, please add it to OS_DOCKER_LOOKUP", test_os
|
||||||
@ -663,6 +786,8 @@ if args.command == "branch":
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
validate_top_level_directory("mongo-binaries.tgz")
|
||||||
|
|
||||||
if not args.skip_enterprise_check:
|
if not args.skip_enterprise_check:
|
||||||
logging.info(
|
logging.info(
|
||||||
"Checking the source files used to build the binaries, use --skip-enterprise-check to skip this check."
|
"Checking the source files used to build the binaries, use --skip-enterprise-check to skip this check."
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import pathlib
|
|||||||
import platform
|
import platform
|
||||||
import pwd
|
import pwd
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tarfile
|
import tarfile
|
||||||
@ -31,8 +32,12 @@ root.addHandler(stdout_handler)
|
|||||||
root.addHandler(file_handler)
|
root.addHandler(file_handler)
|
||||||
|
|
||||||
DOCKER_SYSTEMCTL_REPO = "https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement"
|
DOCKER_SYSTEMCTL_REPO = "https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement"
|
||||||
SYSTEMCTL_URL = DOCKER_SYSTEMCTL_REPO + "/master/files/docker/systemctl3.py"
|
SYSTEMCTL_URL = (
|
||||||
JOURNALCTL_URL = DOCKER_SYSTEMCTL_REPO + "/master/files/docker/journalctl3.py"
|
DOCKER_SYSTEMCTL_REPO + "/eb2a963a7d8413119b432bcb6151af6076b65f84/files/docker/systemctl3.py"
|
||||||
|
)
|
||||||
|
JOURNALCTL_URL = (
|
||||||
|
DOCKER_SYSTEMCTL_REPO + "/eb2a963a7d8413119b432bcb6151af6076b65f84/files/docker/journalctl3.py"
|
||||||
|
)
|
||||||
|
|
||||||
TestArgs = Dict[str, Union[str, int, List[str]]]
|
TestArgs = Dict[str, Union[str, int, List[str]]]
|
||||||
|
|
||||||
@ -49,11 +54,22 @@ def run_and_log(cmd: str, end_on_error: bool = True):
|
|||||||
|
|
||||||
|
|
||||||
def download_extract_package(package: str) -> List[str]:
|
def download_extract_package(package: str) -> List[str]:
|
||||||
# Use wget here because using urllib we get errors like the following
|
# Handle local files (file:// protocol) - these are pre-downloaded by package_test.py
|
||||||
# https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error
|
# when using --evg-build-id for private artifacts
|
||||||
run_and_log('wget -q "{}"'.format(package))
|
if package.startswith("file://"):
|
||||||
downloaded_file = package.split("/")[-1]
|
local_path = package[7:] # Remove "file://" prefix
|
||||||
if not package.endswith(".tgz"):
|
logging.info("Using local file: %s", local_path)
|
||||||
|
downloaded_file = os.path.basename(local_path)
|
||||||
|
# Copy the file to the current directory if it's not already here
|
||||||
|
if local_path != downloaded_file and local_path != os.path.join(".", downloaded_file):
|
||||||
|
shutil.copy(local_path, downloaded_file)
|
||||||
|
else:
|
||||||
|
# Use wget here because using urllib we get errors like the following
|
||||||
|
# https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error
|
||||||
|
run_and_log('wget -q "{}"'.format(package))
|
||||||
|
downloaded_file = package.split("/")[-1]
|
||||||
|
|
||||||
|
if not downloaded_file.endswith(".tgz"):
|
||||||
return [downloaded_file]
|
return [downloaded_file]
|
||||||
|
|
||||||
extracted_paths = [] # type: List[str]
|
extracted_paths = [] # type: List[str]
|
||||||
|
|||||||
@ -48,6 +48,23 @@ ARCH_CHOICES = ["x86_64", "arm64", "aarch64", "s390x"]
|
|||||||
# Made up names for the flavors of distribution we package for.
|
# Made up names for the flavors of distribution we package for.
|
||||||
DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2", "amazon2023"]
|
DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2", "amazon2023"]
|
||||||
|
|
||||||
|
unexpected_lts_release_series = ("8.2",)
|
||||||
|
|
||||||
|
|
||||||
|
def get_suffix(version, stable_name: str, unstable_name: str) -> str:
|
||||||
|
parts = version.split(".")
|
||||||
|
|
||||||
|
major = int(parts[0])
|
||||||
|
minor = int(parts[1])
|
||||||
|
|
||||||
|
series = f"{major}.{minor}"
|
||||||
|
|
||||||
|
if major >= 5:
|
||||||
|
is_stable_version = minor == 0 or series in unexpected_lts_release_series
|
||||||
|
return stable_name if is_stable_version else unstable_name
|
||||||
|
else:
|
||||||
|
return stable_name if minor % 2 == 0 else unstable_name
|
||||||
|
|
||||||
|
|
||||||
class Spec(object):
|
class Spec(object):
|
||||||
"""Spec class."""
|
"""Spec class."""
|
||||||
@ -104,12 +121,9 @@ class Spec(object):
|
|||||||
# e.g., "1.8.2" < "1.8.10", "1.8.2" < "1.8.2-rc1"
|
# e.g., "1.8.2" < "1.8.10", "1.8.2" < "1.8.2-rc1"
|
||||||
return self.ver > version_string
|
return self.ver > version_string
|
||||||
|
|
||||||
def suffix(self):
|
def suffix(self) -> str:
|
||||||
"""Return suffix."""
|
"""Return suffix."""
|
||||||
if int(self.ver.split(".")[0]) >= 5:
|
return get_suffix(self.ver, "-org", "-org-unstable")
|
||||||
return "-org" if int(self.ver.split(".")[1]) == 0 else "-org-unstable"
|
|
||||||
else:
|
|
||||||
return "-org" if int(self.ver.split(".")[1]) % 2 == 0 else "-org-unstable"
|
|
||||||
|
|
||||||
def prelease(self):
|
def prelease(self):
|
||||||
"""Return pre-release verison suffix."""
|
"""Return pre-release verison suffix."""
|
||||||
|
|||||||
@ -54,11 +54,8 @@ class EnterpriseSpec(packager.Spec):
|
|||||||
"""EnterpriseSpec class."""
|
"""EnterpriseSpec class."""
|
||||||
|
|
||||||
def suffix(self):
|
def suffix(self):
|
||||||
|
return packager.get_suffix(self.ver, "-enterprise", "-enterprise-unstable")
|
||||||
"""Suffix."""
|
"""Suffix."""
|
||||||
if int(self.ver.split(".")[0]) >= 5:
|
|
||||||
return "-enterprise" if int(self.ver.split(".")[1]) == 0 else "-enterprise-unstable"
|
|
||||||
else:
|
|
||||||
return "-enterprise" if int(self.ver.split(".")[1]) % 2 == 0 else "-enterprise-unstable"
|
|
||||||
|
|
||||||
def move_required_contents(self):
|
def move_required_contents(self):
|
||||||
"""Move the required contents to the current working directory.
|
"""Move the required contents to the current working directory.
|
||||||
|
|||||||
@ -28,5 +28,3 @@
|
|||||||
# TODO (SERVER-73632): Delete featureFlagRouterPort.
|
# TODO (SERVER-73632): Delete featureFlagRouterPort.
|
||||||
- featureFlagRouterPort
|
- featureFlagRouterPort
|
||||||
- featureFlagSessionsCollectionCoordinatorOnConfigServer
|
- featureFlagSessionsCollectionCoordinatorOnConfigServer
|
||||||
# TODO (SERVER-101672) : Delete featureFlagOplogSamplingAsyncEnabled.
|
|
||||||
- featureFlagOplogSamplingAsyncEnabled
|
|
||||||
|
|||||||
@ -6,9 +6,6 @@ filters:
|
|||||||
- "README.md":
|
- "README.md":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/devprod-correctness
|
- 10gen/devprod-correctness
|
||||||
- "/generated_suites/*viewless_timeseries.yml":
|
|
||||||
approvers:
|
|
||||||
- 10gen/server-catalog-and-routing
|
|
||||||
- "/generated_suites/*injected_catalog_metadata.yml":
|
- "/generated_suites/*injected_catalog_metadata.yml":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/server-catalog-and-routing
|
- 10gen/server-catalog-and-routing
|
||||||
|
|||||||
@ -3,7 +3,7 @@
|
|||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
# AND REGENERATE THE MATRIX SUITES.
|
||||||
#
|
#
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/aggregation_multiversion_fuzzer_last_continuous.yml
|
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/aggregation_expression_multiversion_fuzzer_deterministic_last_lts.yml
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
||||||
##########################################################
|
##########################################################
|
||||||
executor:
|
executor:
|
||||||
@ -16,9 +16,9 @@ executor:
|
|||||||
clusterType: standalone
|
clusterType: standalone
|
||||||
internalQueryAppendIdToSetWindowFieldsSort: true
|
internalQueryAppendIdToSetWindowFieldsSort: true
|
||||||
internalQueryMaxAllowedDensifyDocs: 1000
|
internalQueryMaxAllowedDensifyDocs: 1000
|
||||||
mongosBinVersion: last-continuous
|
mongosBinVersion: last-lts
|
||||||
traceExceptions: false
|
traceExceptions: false
|
||||||
useRandomBinVersionsWithinReplicaSet: last-continuous
|
useRandomBinVersionsWithinReplicaSet: last-lts
|
||||||
nodb: ""
|
nodb: ""
|
||||||
matrix_suite: true
|
matrix_suite: true
|
||||||
selector:
|
selector:
|
||||||
@ -1,27 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/aggregation_expression_multiversion_fuzzer_last_continuous.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests: true
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
clusterType: standalone
|
|
||||||
internalQueryAppendIdToSetWindowFieldsSort: true
|
|
||||||
internalQueryMaxAllowedDensifyDocs: 1000
|
|
||||||
mongosBinVersion: last-continuous
|
|
||||||
traceExceptions: false
|
|
||||||
useRandomBinVersionsWithinReplicaSet: last-continuous
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
roots:
|
|
||||||
- jstestfuzz/out/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/aggregation_multiversion_fuzzer_deterministic_last_continuous.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests: true
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
clusterType: standalone
|
|
||||||
internalQueryAppendIdToSetWindowFieldsSort: true
|
|
||||||
internalQueryMaxAllowedDensifyDocs: 1000
|
|
||||||
mongosBinVersion: last-continuous
|
|
||||||
traceExceptions: false
|
|
||||||
useRandomBinVersionsWithinReplicaSet: last-continuous
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
roots:
|
|
||||||
- jstestfuzz/out/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/concurrency_replication_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- RunDBCheckInBackground
|
|
||||||
- CheckReplDBHashInBackground
|
|
||||||
- ValidateCollectionsInBackground
|
|
||||||
- CheckReplDBHash
|
|
||||||
- ValidateCollections
|
|
||||||
tests: true
|
|
||||||
config: {}
|
|
||||||
fixture:
|
|
||||||
class: ReplicaSetFixture
|
|
||||||
mongod_options:
|
|
||||||
oplogSize: 1024
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
queryAnalysisSamplerConfigurationRefreshSecs: 1
|
|
||||||
queryAnalysisWriterIntervalSecs: 1
|
|
||||||
roleGraphInvalidationIsFatal: 1
|
|
||||||
num_nodes: 3
|
|
||||||
hooks:
|
|
||||||
- class: RunDBCheckInBackground
|
|
||||||
- class: CheckReplDBHashInBackground
|
|
||||||
- class: ValidateCollectionsInBackground
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CleanupConcurrencyWorkloads
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- requires_standalone
|
|
||||||
- requires_sharding
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*/**/*.js
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*.js
|
|
||||||
test_kind: fsm_workload_test
|
|
||||||
@ -1,68 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/concurrency_sharded_replication_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- CheckReplDBHashInBackground
|
|
||||||
- CheckReplDBHash
|
|
||||||
- CheckMetadataConsistencyInBackground
|
|
||||||
- ValidateCollections
|
|
||||||
tests: true
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
eval: await import("jstests/libs/override_methods/implicitly_shard_accessed_collections.js");
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
implicitlyShardOnCreateCollectionOnly: true
|
|
||||||
runningWithBalancer: false
|
|
||||||
fixture:
|
|
||||||
class: ShardedClusterFixture
|
|
||||||
enable_balancer: false
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
queryAnalysisSamplerConfigurationRefreshSecs: 1
|
|
||||||
queryAnalysisWriterIntervalSecs: 1
|
|
||||||
roleGraphInvalidationIsFatal: 1
|
|
||||||
mongos_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
queryAnalysisSamplerConfigurationRefreshSecs: 1
|
|
||||||
num_mongos: 2
|
|
||||||
num_rs_nodes_per_shard: 3
|
|
||||||
num_shards: 2
|
|
||||||
shard_options:
|
|
||||||
mongod_options:
|
|
||||||
oplogSize: 1024
|
|
||||||
hooks:
|
|
||||||
- class: CheckShardFilteringMetadata
|
|
||||||
- class: CheckReplDBHashInBackground
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: CheckMetadataConsistencyInBackground
|
|
||||||
- class: CheckOrphansDeleted
|
|
||||||
- class: CheckRoutingTableConsistency
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CleanupConcurrencyWorkloads
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- requires_standalone
|
|
||||||
- requires_replication
|
|
||||||
- assumes_against_mongod_not_mongos
|
|
||||||
- assumes_balancer_on
|
|
||||||
- requires_profiling
|
|
||||||
- assumes_unsharded_collection
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*/**/*.js
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*.js
|
|
||||||
test_kind: fsm_workload_test
|
|
||||||
@ -1,129 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/no_passthrough_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
description:
|
|
||||||
'"Passthrough" means running a test against different runtime Cluster
|
|
||||||
|
|
||||||
configurations, including topology, runtime flags, fault injections, and other
|
|
||||||
|
|
||||||
parameters. Most tests by default are able to run in "passthrough" suites.
|
|
||||||
|
|
||||||
NoPassthrough is an exception, where tests here only run in the exact
|
|
||||||
|
|
||||||
configuration predefined in the tests themselves.
|
|
||||||
|
|
||||||
'
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests:
|
|
||||||
- jstests/noPassthrough/backup*.js
|
|
||||||
- jstests/noPassthrough/oplog_writes_only_permitted_on_standalone.js
|
|
||||||
- jstests/noPassthrough/wt_unclean_shutdown.js
|
|
||||||
- src/mongo/db/modules/*/jstests/live_restore/*.js
|
|
||||||
- src/mongo/db/modules/enterprise/jstests/hot_backups/**/*.js
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
setParameters:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
setParametersMongos:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/noPassthrough/catalog/server_status_catalog_stats.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/catalog_shard.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/move_collection_create_options.js
|
|
||||||
- jstests/noPassthrough/crud/vectored_insert_no_oplog_hole.js
|
|
||||||
- jstests/noPassthrough/global_catalog/shareded_data_distribution_orphan_docs.js
|
|
||||||
- jstests/noPassthrough/index_builds/create_indexes_return_on_start.js
|
|
||||||
- jstests/noPassthrough/move_collection_not_tracked.js
|
|
||||||
- jstests/noPassthrough/query/change_streams/change_streams_timeseries_shard_collection_event.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/bucket_unpacking_with_sort_granularity_change.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/timeseries_validate_mixed_schema_bucket.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/timeseries_validate_mixed_schema_bucket_legacy_flag.js
|
|
||||||
- jstests/noPassthrough/timeseries/bucket_reopening/timeseries_direct_remove_reopen.js
|
|
||||||
- jstests/noPassthrough/timeseries/bucket_reopening/timeseries_reopen_uncompressed_bucket_for_compression.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/initial_sync_recreate_timeseries_collection.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_block_compressor_options.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_server_parameters.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_startup.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/intermediate_data_consistency_checks_skips.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_insert_compression_failure.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_insert_decompression_failure.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_partial_compressed_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_bucket_spanning_epoch.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_extended_range_rollback.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_extended_range_startup.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/hybrid_index_timeseries.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/resumable_timeseries_index_build_collection_scan_phase.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/timeseries_create_index_option_defaults.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_concurrent.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_block_metrics.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_bson_types.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_count_min_max_sharded.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_spill_metrics.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_topN_bottomN.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_id_filtered_explicit_upack_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_internal_bounded_sort_spilling.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_nested_elem_match.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_query_knob_sbe.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sample_on_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sort.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sparse_fields.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_collStats.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_latency_stats.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_serverStatus.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_server_status_state_management.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_buckets_oplog_update.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_resharding_disabled.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_write_concurrent_collection_replace.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_expire.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_expires_with_partial_index.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_ttl.js
|
|
||||||
- jstests/noPassthrough/timeseries/updates_deletes/timeseries_direct_update.js
|
|
||||||
- jstests/noPassthrough/timeseries/updates_deletes/timeseries_updates_create_compressed_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_compressed_bucket_with_time_out_of_order.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucket_max_span.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucket_reopening.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucketing_parameters_change.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_count.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_data_indexes.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_id_timestamp.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_minmax.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_version.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_v3_buckets_are_unsorted.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/promoting_compressed_sorted_bucket_to_compressed_unsorted_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_limit_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_max_size_min_limit.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_oids.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_closed_due_to_schema_changes.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_compressed_bucket_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_dynamic_bucket_sizing.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_dynamic_bucket_sizing_large.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_find_and_modify_without_shard_key_raw_data.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_idle_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_after_cycle_primary.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_after_failed_insert.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_invalid_timefield.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_ordered_false.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_ordered_true.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_rollback.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_large_measurements.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_large_measurements_max_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_retry_writes.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_retryable_writes_restart.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_time_backward.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_write_concern.js
|
|
||||||
- jstests/noPassthrough/versioning_protocol/stale_shard_version_with_timeseries.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,59 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/replica_sets_jscore_passthrough_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- RunDBCheckInBackground
|
|
||||||
- CheckReplDBHashInBackground
|
|
||||||
- ValidateCollectionsInBackground
|
|
||||||
- CheckReplDBHash
|
|
||||||
- CheckReplOplogs
|
|
||||||
- ValidateCollections
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
eval: globalThis.testingReplication = true;
|
|
||||||
fixture:
|
|
||||||
class: ReplicaSetFixture
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
num_nodes: 2
|
|
||||||
hooks:
|
|
||||||
- class: RunDBCheckInBackground
|
|
||||||
- class: CheckReplDBHashInBackground
|
|
||||||
- class: ValidateCollectionsInBackground
|
|
||||||
- class: CheckReplOplogs
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CleanEveryN
|
|
||||||
n: 20
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_files:
|
|
||||||
- jstests/core/txns/abort_expired_transaction.js
|
|
||||||
- jstests/core/txns/abort_transaction_thread_does_not_block_on_locks.js
|
|
||||||
- jstests/core/txns/kill_op_on_txn_expiry.js
|
|
||||||
- jstests/core/**/set_param1.js
|
|
||||||
- jstests/core/query/awaitdata_getmore_cmd.js
|
|
||||||
- jstests/core/administrative/current_op/currentop.js
|
|
||||||
- jstests/core/administrative/fsync/fsync.js
|
|
||||||
- jstests/core/txns/prepare_conflict.js
|
|
||||||
- jstests/core/txns/prepare_conflict_aggregation_behavior.js
|
|
||||||
- jstests/core/timeseries/write/timeseries_update_multi.js
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- assumes_standalone_mongod
|
|
||||||
- requires_profiling
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/core/**/*.js
|
|
||||||
- jstests/fle2/**/*.js
|
|
||||||
- src/mongo/db/modules/*/jstests/fle2/**/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,88 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/sharded_collections_jscore_passthrough_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- CheckReplDBHash
|
|
||||||
- CheckMetadataConsistencyInBackground
|
|
||||||
- ValidateCollections
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
eval: await import("jstests/libs/override_methods/implicitly_shard_accessed_collections.js");
|
|
||||||
fixture:
|
|
||||||
class: ShardedClusterFixture
|
|
||||||
enable_balancer: false
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
mongos_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
num_shards: 2
|
|
||||||
hooks:
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: CheckMetadataConsistencyInBackground
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CheckOrphansDeleted
|
|
||||||
- class: CleanEveryN
|
|
||||||
n: 20
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_files:
|
|
||||||
- jstests/core/txns/**/*.js
|
|
||||||
- jstests/core/**/apitest_db.js
|
|
||||||
- jstests/core/**/awaitdata_getmore_cmd.js
|
|
||||||
- jstests/core/**/bypass_doc_validation.js
|
|
||||||
- jstests/core/**/check_shard_index.js
|
|
||||||
- jstests/core/**/compact_keeps_indexes.js
|
|
||||||
- jstests/core/**/currentop.js
|
|
||||||
- jstests/core/**/dbhash.js
|
|
||||||
- jstests/core/**/fsync.js
|
|
||||||
- jstests/core/**/geo_s2cursorlimitskip.js
|
|
||||||
- jstests/core/**/geo_update_btree2.js
|
|
||||||
- jstests/core/**/queryoptimizera.js
|
|
||||||
- jstests/core/**/startup_log.js
|
|
||||||
- jstests/core/**/tailable_cursor_invalidation.js
|
|
||||||
- jstests/core/**/tailable_getmore_batch_size.js
|
|
||||||
- jstests/core/**/tailable_skip_limit.js
|
|
||||||
- jstests/core/**/query/top/top.js
|
|
||||||
- jstests/core/**/geo_2d_explain.js
|
|
||||||
- jstests/core/**/geo_s2explain.js
|
|
||||||
- jstests/core/**/geo_s2sparse.js
|
|
||||||
- jstests/core/**/operation_latency_histogram.js
|
|
||||||
- jstests/core/**/apitest_dbcollection.js
|
|
||||||
- jstests/core/**/bad_index_plugin.js
|
|
||||||
- jstests/core/**/create_indexes.js
|
|
||||||
- jstests/core/**/list_indexes_non_existent_ns.js
|
|
||||||
- jstests/core/**/mr_preserve_indexes.js
|
|
||||||
- jstests/core/**/distinct_index1.js
|
|
||||||
- jstests/core/**/expr_index_use.js
|
|
||||||
- jstests/core/**/index_multikey.js
|
|
||||||
- jstests/core/**/query/explain/optimized_match_explain.js
|
|
||||||
- jstests/core/**/sort_array.js
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- assumes_standalone_mongod
|
|
||||||
- assumes_against_mongod_not_mongos
|
|
||||||
- assumes_no_implicit_collection_creation_on_get_collection
|
|
||||||
- assumes_no_implicit_collection_creation_after_drop
|
|
||||||
- assumes_no_implicit_index_creation
|
|
||||||
- assumes_unsharded_collection
|
|
||||||
- cannot_create_unique_index_when_using_hashed_shard_key
|
|
||||||
- requires_profiling
|
|
||||||
- requires_capped
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/core/**/*.js
|
|
||||||
- jstests/core_sharding/**/*.js
|
|
||||||
- jstests/fle2/**/*.js
|
|
||||||
- src/mongo/db/modules/*/jstests/fle2/**/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,66 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/sharding_jscore_passthrough_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- CheckReplDBHash
|
|
||||||
- CheckMetadataConsistencyInBackground
|
|
||||||
- ValidateCollections
|
|
||||||
config:
|
|
||||||
shell_options: {}
|
|
||||||
fixture:
|
|
||||||
class: ShardedClusterFixture
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
mongos_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
num_rs_nodes_per_shard: 1
|
|
||||||
num_shards: 2
|
|
||||||
hooks:
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: CheckMetadataConsistencyInBackground
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CheckOrphansDeleted
|
|
||||||
- class: CleanEveryN
|
|
||||||
n: 20
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_files:
|
|
||||||
- jstests/core/txns/**/*.js
|
|
||||||
- jstests/core/**/apitest_db.js
|
|
||||||
- jstests/core/**/check_shard_index.js
|
|
||||||
- jstests/core/**/compact_keeps_indexes.js
|
|
||||||
- jstests/core/**/currentop.js
|
|
||||||
- jstests/core/**/dbhash.js
|
|
||||||
- jstests/core/**/fsync.js
|
|
||||||
- jstests/core/**/geo_s2cursorlimitskip.js
|
|
||||||
- jstests/core/**/geo_update_btree2.js
|
|
||||||
- jstests/core/**/queryoptimizera.js
|
|
||||||
- jstests/core/**/startup_log.js
|
|
||||||
- jstests/core/**/query/top/top.js
|
|
||||||
- jstests/core/**/geo_2d_explain.js
|
|
||||||
- jstests/core/**/geo_s2explain.js
|
|
||||||
- jstests/core/**/geo_s2sparse.js
|
|
||||||
- jstests/core/**/operation_latency_histogram.js
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- assumes_standalone_mongod
|
|
||||||
- assumes_against_mongod_not_mongos
|
|
||||||
- requires_profiling
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/core/**/*.js
|
|
||||||
- jstests/core_sharding/**/*.js
|
|
||||||
- jstests/fle2/**/*.js
|
|
||||||
- src/mongo/db/modules/*/jstests/fle2/**/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,60 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/sharding_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests:
|
|
||||||
- jstests/sharding/*reshard*.js
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
setParameters:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
setParametersMongos:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/sharding/analyze_shard_key/timeseries.js
|
|
||||||
- jstests/sharding/libs/timeseries_update_multi_util.js
|
|
||||||
- jstests/sharding/resharding_timeseries/move_timeseries.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_disallow_writes.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_nonempty_stash.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_stash_resolution.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_ttl_deletes.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_validation.js
|
|
||||||
- jstests/sharding/resharding_timeseries/resharding_ts_resume_agg_token.js
|
|
||||||
- jstests/sharding/resharding_timeseries/unshard_timeseries.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_balancer.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_buckets_modification_with_id.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_cluster_collstats.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_cluster_indexstats.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_coll_mod.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_coll_mod_bucketing_parameters.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_drop.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_indexes.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert_move_collection.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert_targeting_normalize_metadata.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_multiple_mongos.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_orphan_buckets.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_out_conflicting_namespace_on_shard.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_query.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_query_extended_range.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_shard_collection.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_shard_collection_with_deprecation_message.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_sharding_admin_commands.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_time_value_rounding.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_user_system_buckets_sharding.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,83 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/unsplittable_collections_created_on_any_shard_jscore_passthrough_viewless_timeseries.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
hooks:
|
|
||||||
- CheckReplDBHash
|
|
||||||
- CheckMetadataConsistencyInBackground
|
|
||||||
- ValidateCollections
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
createsUnsplittableCollectionsOnRandomShards: true
|
|
||||||
implicitlyTrackUnshardedCollectionOnCreation: true
|
|
||||||
fixture:
|
|
||||||
class: ShardedClusterFixture
|
|
||||||
enable_balancer: false
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
failpoint.createUnshardedCollectionRandomizeDataShard:
|
|
||||||
mode: alwaysOn
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
mongos_options:
|
|
||||||
set_parameters:
|
|
||||||
enableTestCommands: 1
|
|
||||||
failpoint.createUnshardedCollectionRandomizeDataShard:
|
|
||||||
mode: alwaysOn
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
num_shards: 2
|
|
||||||
hooks:
|
|
||||||
- class: CheckReplDBHash
|
|
||||||
- class: CheckMetadataConsistencyInBackground
|
|
||||||
- class: ValidateCollections
|
|
||||||
- class: CheckOrphansDeleted
|
|
||||||
- class: CleanEveryN
|
|
||||||
n: 20
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
exclude_files:
|
|
||||||
- jstests/core/txns/**/*.js
|
|
||||||
- jstests/core/**/apitest_db.js
|
|
||||||
- jstests/core/**/check_shard_index.js
|
|
||||||
- jstests/core/**/compact_keeps_indexes.js
|
|
||||||
- jstests/core/**/currentop.js
|
|
||||||
- jstests/core/**/dbhash.js
|
|
||||||
- jstests/core/**/fsync.js
|
|
||||||
- jstests/core/**/geo_s2cursorlimitskip.js
|
|
||||||
- jstests/core/**/geo_update_btree2.js
|
|
||||||
- jstests/core/**/queryoptimizera.js
|
|
||||||
- jstests/core/**/startup_log.js
|
|
||||||
- jstests/core/**/query/top/top.js
|
|
||||||
- jstests/core/**/geo_2d_explain.js
|
|
||||||
- jstests/core/**/geo_s2explain.js
|
|
||||||
- jstests/core/**/geo_s2sparse.js
|
|
||||||
- jstests/core/**/operation_latency_histogram.js
|
|
||||||
- jstests/core/**/or_to_in.js
|
|
||||||
- jstests/core/query/explain/explain_find_trivially_false_predicates_in_agg_pipelines.js
|
|
||||||
- jstests/core/query/query_settings/query_settings_index_application_aggregate.js
|
|
||||||
- jstests/core/write/insert/insert_id_undefined.js
|
|
||||||
- jstests/core/timeseries/ddl/timeseries_user_system_buckets.js
|
|
||||||
- jstests/core/ddl/clone_collection_as_capped.js
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- assumes_standalone_mongod
|
|
||||||
- assumes_against_mongod_not_mongos
|
|
||||||
- requires_profiling
|
|
||||||
- requires_system_dot_js_stored_functions
|
|
||||||
- assumes_no_track_upon_creation
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
roots:
|
|
||||||
- jstests/core/**/*.js
|
|
||||||
- jstests/core_sharding/**/*.js
|
|
||||||
- jstests/fle2/**/*.js
|
|
||||||
- src/mongo/db/modules/*/jstests/fle2/**/*.js
|
|
||||||
- jstests/libs/random_data_shard_suite_selftest.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -3,7 +3,7 @@
|
|||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
# AND REGENERATE THE MATRIX SUITES.
|
||||||
#
|
#
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/update_fuzzer_last_continuous.yml
|
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/update_fuzzer_deterministic_last_lts.yml
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
||||||
##########################################################
|
##########################################################
|
||||||
executor:
|
executor:
|
||||||
@ -1,22 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/update_fuzzer_replication_deterministic_last_continuous.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests: true
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
clusterType: replset
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
roots:
|
|
||||||
- jstestfuzz/out/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -1,22 +0,0 @@
|
|||||||
##########################################################
|
|
||||||
# THIS IS A GENERATED FILE -- DO NOT MODIFY.
|
|
||||||
# IF YOU WISH TO MODIFY THIS SUITE, MODIFY THE CORRESPONDING MATRIX SUITE MAPPING FILE
|
|
||||||
# AND REGENERATE THE MATRIX SUITES.
|
|
||||||
#
|
|
||||||
# matrix suite mapping file: buildscripts/resmokeconfig/matrix_suites/mappings/update_fuzzer_replication_last_continuous.yml
|
|
||||||
# regenerate matrix suites: buildscripts/resmoke.py generate-matrix-suites && bazel run //:format
|
|
||||||
##########################################################
|
|
||||||
executor:
|
|
||||||
archive:
|
|
||||||
tests: true
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
clusterType: replset
|
|
||||||
nodb: ""
|
|
||||||
matrix_suite: true
|
|
||||||
selector:
|
|
||||||
roots:
|
|
||||||
- jstestfuzz/out/*.js
|
|
||||||
test_kind: js_test
|
|
||||||
@ -47,9 +47,6 @@ filters:
|
|||||||
approvers:
|
approvers:
|
||||||
- 10gen/query-execution-query-settings
|
- 10gen/query-execution-query-settings
|
||||||
- 10gen/query-integration-observability
|
- 10gen/query-integration-observability
|
||||||
- "*viewless_timeseries*":
|
|
||||||
approvers:
|
|
||||||
- 10gen/server-catalog-and-routing
|
|
||||||
- "sharding_pqs*":
|
- "sharding_pqs*":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/query-execution-query-settings
|
- 10gen/query-execution-query-settings
|
||||||
|
|||||||
@ -0,0 +1,3 @@
|
|||||||
|
base_suite: generational_fuzzer
|
||||||
|
overrides:
|
||||||
|
- "multiversion.replica_sets_multiversion_testdata_last_lts"
|
||||||
@ -1,3 +0,0 @@
|
|||||||
base_suite: generational_fuzzer
|
|
||||||
overrides:
|
|
||||||
- "multiversion.replica_sets_multiversion_testdata_last_continuous"
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
base_suite: generational_fuzzer
|
|
||||||
overrides:
|
|
||||||
- "multiversion.replica_sets_multiversion_testdata_last_continuous"
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
base_suite: generational_fuzzer
|
|
||||||
overrides:
|
|
||||||
- "multiversion.replica_sets_multiversion_testdata_last_continuous"
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/concurrency/fsm_workloads/timeseries/* tests in concurrency_replication suite suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: concurrency_replication
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.all_concurrency_timeseries_tests_selector"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/concurrency/fsm_workloads/timeseries/* tests in concurrency_sharded_replication suite suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: concurrency_sharded_replication
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.all_concurrency_timeseries_tests_selector"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
# Temporary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all timeseries tests in the noPassthrough suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temporary suite and all its usages.
|
|
||||||
base_suite: no_passthrough
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.only_validated_no_passthrough_timeseries_tests_selector"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_testdata_parameter_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_testdata_parameter_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/core/timeseries/* tests in replicaset jscore pasthrough suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: replica_sets_jscore_passthrough
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/core/timeseries/* tests in sharded jscore pasthrough suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: sharded_collections_jscore_passthrough
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/core/timeseries/* tests in sharding jscore pasthrough suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: sharding_jscore_passthrough
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
# Temporary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all timeseries tests in the sharding suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temporary suite and all its usages.
|
|
||||||
base_suite: sharding
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.only_validated_sharding_timeseries_tests_selector"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_testdata_parameter_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_testdata_parameter_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
# Temorary suites to assists the development of new viewless timeseries collection.
|
|
||||||
# Run all jstests/core/timeseries/* tests in sharding jscore pasthrough suite with
|
|
||||||
# CreateViewlessTimeseriesCollection feature flag enabled.
|
|
||||||
#
|
|
||||||
# TODO SERVER-101590: remove this temorary suite and all its usages.
|
|
||||||
base_suite: unsplittable_collections_created_on_any_shard_jscore_passthrough
|
|
||||||
overrides:
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongod"
|
|
||||||
- "viewless_timeseries.enable_viewless_timeseries_feature_flag_mongos"
|
|
||||||
|
|
||||||
excludes:
|
|
||||||
- "viewless_timeseries.exclude_viewless_timeseries_incompabile_tag"
|
|
||||||
@ -1 +0,0 @@
|
|||||||
base_suite: update_fuzzer_replication
|
|
||||||
@ -1 +0,0 @@
|
|||||||
base_suite: update_fuzzer_replication
|
|
||||||
@ -24,9 +24,6 @@ filters:
|
|||||||
- "repeat_queries.yml":
|
- "repeat_queries.yml":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/query-optimization
|
- 10gen/query-optimization
|
||||||
- "*viewless_timeseries.yml":
|
|
||||||
approvers:
|
|
||||||
- 10gen/server-catalog-and-routing
|
|
||||||
- "stepdown_primary_change_streams.yml":
|
- "stepdown_primary_change_streams.yml":
|
||||||
approvers:
|
approvers:
|
||||||
- 10gen/query-execution-change-streams
|
- 10gen/query-execution-change-streams
|
||||||
|
|||||||
@ -1,194 +0,0 @@
|
|||||||
- name: enable_viewless_timeseries_feature_flag_mongod
|
|
||||||
value:
|
|
||||||
executor:
|
|
||||||
fixture:
|
|
||||||
mongod_options:
|
|
||||||
set_parameters:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
- name: enable_viewless_timeseries_feature_flag_mongos
|
|
||||||
value:
|
|
||||||
executor:
|
|
||||||
fixture:
|
|
||||||
mongos_options:
|
|
||||||
set_parameters:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: 1
|
|
||||||
- name: enable_viewless_timeseries_feature_flag_testdata_parameter_mongod
|
|
||||||
value:
|
|
||||||
executor:
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
setParameters:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
- name: enable_viewless_timeseries_feature_flag_testdata_parameter_mongos
|
|
||||||
value:
|
|
||||||
executor:
|
|
||||||
config:
|
|
||||||
shell_options:
|
|
||||||
global_vars:
|
|
||||||
TestData:
|
|
||||||
setParametersMongos:
|
|
||||||
featureFlagCreateViewlessTimeseriesCollections: true
|
|
||||||
- name: all_concurrency_timeseries_tests_selector
|
|
||||||
value:
|
|
||||||
selector:
|
|
||||||
exclude_files: null
|
|
||||||
roots:
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*/**/*.js
|
|
||||||
- jstests/concurrency/fsm_workloads/**/*timeseries*.js
|
|
||||||
- name: all_sharding_timeseries_tests_selector
|
|
||||||
value:
|
|
||||||
selector:
|
|
||||||
exclude_files: null
|
|
||||||
roots:
|
|
||||||
- jstests/sharding/*timeseries*/*.js
|
|
||||||
- jstests/sharding/**/*timeseries*.js
|
|
||||||
- name: only_validated_sharding_timeseries_tests_selector
|
|
||||||
value:
|
|
||||||
selector:
|
|
||||||
exclude_files: null
|
|
||||||
roots:
|
|
||||||
- jstests/sharding/analyze_shard_key/timeseries.js
|
|
||||||
- jstests/sharding/libs/timeseries_update_multi_util.js
|
|
||||||
- jstests/sharding/resharding_timeseries/move_timeseries.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_disallow_writes.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_nonempty_stash.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_stash_resolution.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_ttl_deletes.js
|
|
||||||
- jstests/sharding/resharding_timeseries/reshard_timeseries_validation.js
|
|
||||||
- jstests/sharding/resharding_timeseries/resharding_ts_resume_agg_token.js
|
|
||||||
- jstests/sharding/resharding_timeseries/unshard_timeseries.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_balancer.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_buckets_modification_with_id.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_cluster_collstats.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_cluster_indexstats.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_coll_mod.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_coll_mod_bucketing_parameters.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_drop.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_indexes.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert_move_collection.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_insert_targeting_normalize_metadata.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_multiple_mongos.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_orphan_buckets.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_out_conflicting_namespace_on_shard.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_query.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_query_extended_range.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_shard_collection.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_shard_collection_with_deprecation_message.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_sharding_admin_commands.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_time_value_rounding.js
|
|
||||||
- jstests/sharding/timeseries/timeseries_user_system_buckets_sharding.js
|
|
||||||
- name: all_no_passthrough_timeseries_tests_selector
|
|
||||||
value:
|
|
||||||
selector:
|
|
||||||
exclude_files: null
|
|
||||||
roots:
|
|
||||||
- jstests/noPassthrough/catalog/server_status_catalog_stats.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/catalog_shard.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/move_collection_create_options.js
|
|
||||||
- jstests/noPassthrough/crud/bulk_write_metrics.js
|
|
||||||
- jstests/noPassthrough/crud/vectored_insert_no_oplog_hole.js
|
|
||||||
- jstests/noPassthrough/global_catalog/shareded_data_distribution_orphan_docs.js
|
|
||||||
- jstests/noPassthrough/index_builds/create_indexes_return_on_start.js
|
|
||||||
- jstests/noPassthrough/logging/log_collectionType.js
|
|
||||||
- jstests/noPassthrough/move_collection_not_tracked.js
|
|
||||||
- jstests/noPassthrough/**/timeseries/**/*.js
|
|
||||||
- jstests/noPassthrough/**/*timeseries*.js
|
|
||||||
- name: only_validated_no_passthrough_timeseries_tests_selector
|
|
||||||
value:
|
|
||||||
selector:
|
|
||||||
exclude_files: null
|
|
||||||
roots:
|
|
||||||
- jstests/noPassthrough/catalog/server_status_catalog_stats.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/catalog_shard.js
|
|
||||||
- jstests/noPassthrough/cluster_scalability_misc/move_collection_create_options.js
|
|
||||||
- jstests/noPassthrough/crud/vectored_insert_no_oplog_hole.js
|
|
||||||
- jstests/noPassthrough/global_catalog/shareded_data_distribution_orphan_docs.js
|
|
||||||
- jstests/noPassthrough/index_builds/create_indexes_return_on_start.js
|
|
||||||
- jstests/noPassthrough/move_collection_not_tracked.js
|
|
||||||
- jstests/noPassthrough/query/change_streams/change_streams_timeseries_shard_collection_event.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/bucket_unpacking_with_sort_granularity_change.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/timeseries_validate_mixed_schema_bucket.js
|
|
||||||
- jstests/noPassthrough/query/timeseries/timeseries_validate_mixed_schema_bucket_legacy_flag.js
|
|
||||||
- jstests/noPassthrough/timeseries/bucket_reopening/timeseries_direct_remove_reopen.js
|
|
||||||
- jstests/noPassthrough/timeseries/bucket_reopening/timeseries_reopen_uncompressed_bucket_for_compression.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/initial_sync_recreate_timeseries_collection.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_block_compressor_options.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_server_parameters.js
|
|
||||||
- jstests/noPassthrough/timeseries/create/timeseries_startup.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/intermediate_data_consistency_checks_skips.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_insert_compression_failure.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_insert_decompression_failure.js
|
|
||||||
- jstests/noPassthrough/timeseries/data_integrity/timeseries_partial_compressed_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_bucket_spanning_epoch.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_extended_range_rollback.js
|
|
||||||
- jstests/noPassthrough/timeseries/extended_range/timeseries_extended_range_startup.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/hybrid_index_timeseries.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/resumable_timeseries_index_build_collection_scan_phase.js
|
|
||||||
- jstests/noPassthrough/timeseries/index_builds/timeseries_create_index_option_defaults.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_concurrent.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_block_metrics.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_bson_types.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_count_min_max_sharded.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_spill_metrics.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_group_topN_bottomN.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_id_filtered_explicit_upack_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_internal_bounded_sort_spilling.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_nested_elem_match.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_query_knob_sbe.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sample_on_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sort.js
|
|
||||||
- jstests/noPassthrough/timeseries/query/timeseries_sparse_fields.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_collStats.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_latency_stats.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_serverStatus.js
|
|
||||||
- jstests/noPassthrough/timeseries/stats/timeseries_server_status_state_management.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_buckets_oplog_update.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_resharding_disabled.js
|
|
||||||
- jstests/noPassthrough/timeseries/timeseries_write_concurrent_collection_replace.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_expire.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_expires_with_partial_index.js
|
|
||||||
- jstests/noPassthrough/timeseries/ttl/timeseries_ttl.js
|
|
||||||
- jstests/noPassthrough/timeseries/updates_deletes/timeseries_direct_update.js
|
|
||||||
- jstests/noPassthrough/timeseries/updates_deletes/timeseries_updates_create_compressed_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_compressed_bucket_with_time_out_of_order.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucket_max_span.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucket_reopening.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_bucketing_parameters_change.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_count.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_data_indexes.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_id_timestamp.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_minmax.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_timeseries_version.js
|
|
||||||
- jstests/noPassthrough/timeseries/validate/validate_v3_buckets_are_unsorted.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/promoting_compressed_sorted_bucket_to_compressed_unsorted_bucket.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_limit_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_max_size_min_limit.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_bucket_oids.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_closed_due_to_schema_changes.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_compressed_bucket_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_dynamic_bucket_sizing.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_dynamic_bucket_sizing_large.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_find_and_modify_without_shard_key_raw_data.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_idle_buckets.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_after_cycle_primary.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_after_failed_insert.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_invalid_timefield.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_ordered_false.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_ordered_true.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_insert_rollback.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_large_measurements.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_large_measurements_max_size.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_retry_writes.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_retryable_writes_restart.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_time_backward.js
|
|
||||||
- jstests/noPassthrough/timeseries/write/timeseries_write_concern.js
|
|
||||||
- jstests/noPassthrough/versioning_protocol/stale_shard_version_with_timeseries.js
|
|
||||||
- name: exclude_viewless_timeseries_incompabile_tag
|
|
||||||
value:
|
|
||||||
exclude_with_any_tags:
|
|
||||||
- viewless_timeseries_incompatible
|
|
||||||
- does_not_support_viewless_timeseries_yet
|
|
||||||
@ -22,6 +22,9 @@ selector:
|
|||||||
# These workloads use a verbose log level.
|
# These workloads use a verbose log level.
|
||||||
- jstests/concurrency/fsm_workloads/ddl/rename_collection/collection_uuid.js
|
- jstests/concurrency/fsm_workloads/ddl/rename_collection/collection_uuid.js
|
||||||
|
|
||||||
|
# TODO BACKPORT-25689: reenable this workload after backport to 8.0.
|
||||||
|
- jstests/concurrency/fsm_workloads/query/remove/update_and_batched_delete.js
|
||||||
|
|
||||||
exclude_with_any_tags:
|
exclude_with_any_tags:
|
||||||
- requires_standalone
|
- requires_standalone
|
||||||
# The ability to shut down a node while its in the middle of applying ops is required for
|
# The ability to shut down a node while its in the middle of applying ops is required for
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
test_kind: js_test
|
test_kind: js_test
|
||||||
|
|
||||||
selector:
|
selector:
|
||||||
|
exclude_files:
|
||||||
|
# TODO: SERVER-109999 Remove this line.
|
||||||
|
- src/mongo/db/modules/enterprise/jstests/external_auth_oidc_azure/oidc_e2e_azure_machine.js
|
||||||
roots:
|
roots:
|
||||||
- src/mongo/db/modules/enterprise/jstests/external_auth_oidc_azure/*.js
|
- src/mongo/db/modules/enterprise/jstests/external_auth_oidc_azure/*.js
|
||||||
|
|
||||||
|
|||||||
@ -53,6 +53,7 @@ selector:
|
|||||||
- jstests/core/query/resume_query_from_non_existent_record.js
|
- jstests/core/query/resume_query_from_non_existent_record.js
|
||||||
- jstests/core/query/resume_query.js
|
- jstests/core/query/resume_query.js
|
||||||
- jstests/core/timeseries/query/timeseries_resume_after.js
|
- jstests/core/timeseries/query/timeseries_resume_after.js
|
||||||
|
- jstests/core/query/start_at_blocked_in_sbe.js
|
||||||
# This test creates collections directly using applyOps commands. So those collections are not
|
# This test creates collections directly using applyOps commands. So those collections are not
|
||||||
# tracked by sharding and therefore not discoverable by listCollections commands that go through
|
# tracked by sharding and therefore not discoverable by listCollections commands that go through
|
||||||
# the router. TODO (SERVER-84342): listCollections command doesn't return collections created
|
# the router. TODO (SERVER-84342): listCollections command doesn't return collections created
|
||||||
|
|||||||
@ -48,6 +48,7 @@ selector:
|
|||||||
- jstests/core/query/resume_query_from_non_existent_record.js
|
- jstests/core/query/resume_query_from_non_existent_record.js
|
||||||
- jstests/core/query/resume_query.js
|
- jstests/core/query/resume_query.js
|
||||||
- jstests/core/timeseries/query/timeseries_resume_after.js
|
- jstests/core/timeseries/query/timeseries_resume_after.js
|
||||||
|
- jstests/core/query/start_at_blocked_in_sbe.js
|
||||||
# This test creates collections directly using applyOps commands. So those collections are not
|
# This test creates collections directly using applyOps commands. So those collections are not
|
||||||
# tracked by sharding and therefore not discoverable by listCollections commands that go through
|
# tracked by sharding and therefore not discoverable by listCollections commands that go through
|
||||||
# the router. TODO (SERVER-84342): listCollections command doesn't return collections created
|
# the router. TODO (SERVER-84342): listCollections command doesn't return collections created
|
||||||
|
|||||||
@ -278,6 +278,7 @@ selector:
|
|||||||
- jstests/sharding/query/change_streams/change_stream_no_shards.js
|
- jstests/sharding/query/change_streams/change_stream_no_shards.js
|
||||||
- jstests/sharding/query/change_streams/change_stream_on_system_collection.js
|
- jstests/sharding/query/change_streams/change_stream_on_system_collection.js
|
||||||
- jstests/sharding/query/change_streams/change_stream_transaction_sharded.js
|
- jstests/sharding/query/change_streams/change_stream_transaction_sharded.js
|
||||||
|
- jstests/sharding/query/change_streams/change_stream_transaction_sharded_commit_timestamp.js
|
||||||
- jstests/sharding/query/change_streams/change_stream_update_lookup_collation.js
|
- jstests/sharding/query/change_streams/change_stream_update_lookup_collation.js
|
||||||
- jstests/sharding/query/change_streams/change_stream_update_lookup_read_concern.js
|
- jstests/sharding/query/change_streams/change_stream_update_lookup_read_concern.js
|
||||||
- jstests/sharding/query/change_streams/change_streams.js
|
- jstests/sharding/query/change_streams/change_streams.js
|
||||||
|
|||||||
@ -268,6 +268,7 @@ selector:
|
|||||||
# TODO(SERVER-85322): Try to include this test(s).
|
# TODO(SERVER-85322): Try to include this test(s).
|
||||||
- jstests/core/query/resume_query.js
|
- jstests/core/query/resume_query.js
|
||||||
- jstests/core/query/resume_query_from_non_existent_record.js
|
- jstests/core/query/resume_query_from_non_existent_record.js
|
||||||
|
- jstests/core/query/start_at_blocked_in_sbe.js
|
||||||
# Inserting into a time-series collection does not replace top-level empty timestamps.
|
# Inserting into a time-series collection does not replace top-level empty timestamps.
|
||||||
- jstests/core/write/ts1.js
|
- jstests/core/write/ts1.js
|
||||||
# Cursor manipulation incompatible with count 'applySkipLimit' option.
|
# Cursor manipulation incompatible with count 'applySkipLimit' option.
|
||||||
|
|||||||
@ -3,7 +3,7 @@
|
|||||||
import collections
|
import collections
|
||||||
import datetime
|
import datetime
|
||||||
import itertools
|
import itertools
|
||||||
import os.path
|
import os
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import buildscripts.resmokelib.setup_multiversion.config as multiversion_config
|
import buildscripts.resmokelib.setup_multiversion.config as multiversion_config
|
||||||
@ -817,3 +817,6 @@ VALIDATE_SELECTOR_PATHS = True
|
|||||||
# If set, resmoke.py will set Testdata.pauseAfterPopulate to allow tests that check this
|
# If set, resmoke.py will set Testdata.pauseAfterPopulate to allow tests that check this
|
||||||
# flag to pause after populating their initial datasets.
|
# flag to pause after populating their initial datasets.
|
||||||
PAUSE_AFTER_POPULATE = None
|
PAUSE_AFTER_POPULATE = None
|
||||||
|
|
||||||
|
# Whether ASAN (AddressSanitizer) is enabled, determined by the presence of ASAN_OPTIONS.
|
||||||
|
IS_ASAN = bool(os.environ.get("ASAN_OPTIONS"))
|
||||||
|
|||||||
@ -56,6 +56,12 @@ If the min/max is not inclusive, this is added as a note above the parameter.
|
|||||||
|
|
||||||
config_fuzzer_params = {
|
config_fuzzer_params = {
|
||||||
"mongod": {
|
"mongod": {
|
||||||
|
"analyzeShardKeyNumRanges": {
|
||||||
|
"min": 2,
|
||||||
|
"max": 100,
|
||||||
|
"period": 5,
|
||||||
|
"fuzz_at": ["startup", "runtime"],
|
||||||
|
},
|
||||||
"analyzeShardKeySplitPointExpirationSecs": {"min": 1, "max": 300, "fuzz_at": ["startup"]},
|
"analyzeShardKeySplitPointExpirationSecs": {"min": 1, "max": 300, "fuzz_at": ["startup"]},
|
||||||
"collectionSamplingLogIntervalSeconds": {
|
"collectionSamplingLogIntervalSeconds": {
|
||||||
"min": 5,
|
"min": 5,
|
||||||
@ -495,6 +501,17 @@ config_fuzzer_params = {
|
|||||||
"period": 5,
|
"period": 5,
|
||||||
"fuzz_at": ["startup", "runtime"],
|
"fuzz_at": ["startup", "runtime"],
|
||||||
},
|
},
|
||||||
|
"preAuthMaximumMessageSizeBytes": {
|
||||||
|
"min": 65536, # 64 KiB
|
||||||
|
"max": 16777216, # 16 MiB
|
||||||
|
"period": 10,
|
||||||
|
"fuzz_at": ["startup", "runtime"],
|
||||||
|
},
|
||||||
|
"capMemoryConsumptionForPreAuthBuffers": {
|
||||||
|
"min": 80,
|
||||||
|
"max": 100,
|
||||||
|
"fuzz_at": ["startup"],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"mongos": {
|
"mongos": {
|
||||||
# We need a higher timeout to account for test slowness
|
# We need a higher timeout to account for test slowness
|
||||||
@ -539,6 +556,17 @@ config_fuzzer_params = {
|
|||||||
"choices": [{"mode": "off"}],
|
"choices": [{"mode": "off"}],
|
||||||
"fuzz_at": ["startup"],
|
"fuzz_at": ["startup"],
|
||||||
},
|
},
|
||||||
|
"preAuthMaximumMessageSizeBytes": {
|
||||||
|
"min": 65536, # 64 KiB
|
||||||
|
"max": 16777216, # 16 MiB
|
||||||
|
"period": 10,
|
||||||
|
"fuzz_at": ["startup", "runtime"],
|
||||||
|
},
|
||||||
|
"capMemoryConsumptionForPreAuthBuffers": {
|
||||||
|
"min": 80,
|
||||||
|
"max": 100,
|
||||||
|
"fuzz_at": ["startup"],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -823,7 +823,7 @@ class GDBDumper(Dumper):
|
|||||||
text=True,
|
text=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
regex = re.search("Core was generated by `(.*)'.", process.stdout)
|
regex = re.search("Core was generated by `(.*)'.", process.stdout, re.DOTALL)
|
||||||
if not regex:
|
if not regex:
|
||||||
raise RuntimeError("gdb output did not match pattern, could not find binary name")
|
raise RuntimeError("gdb output did not match pattern, could not find binary name")
|
||||||
|
|
||||||
|
|||||||
@ -22,20 +22,18 @@ BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml"
|
|||||||
BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo"
|
BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo"
|
||||||
|
|
||||||
|
|
||||||
def get_backports_required_hash_for_shell_version(mongo_shell_path: str | None = None):
|
def get_backports_required_hash(mongod_path: str | None = None):
|
||||||
"""Parse the old shell binary to get the commit hash."""
|
"""Parse the old binary to get the commit hash."""
|
||||||
env_vars = os.environ.copy()
|
env_vars = os.environ.copy()
|
||||||
paths = get_path_env_var(env_vars=env_vars)
|
paths = get_path_env_var(env_vars=env_vars)
|
||||||
env_vars["PATH"] = os.pathsep.join(paths)
|
env_vars["PATH"] = os.pathsep.join(paths)
|
||||||
|
|
||||||
mongo_shell = mongo_shell_path
|
mongod = mongod_path
|
||||||
if is_windows():
|
if is_windows():
|
||||||
mongo_shell = mongo_shell_path + ".exe"
|
mongod = mongod_path + ".exe"
|
||||||
|
|
||||||
shell_version = check_output(f"{mongo_shell} --version", shell=True, env=env_vars).decode(
|
version = check_output(f"{mongod} --version", shell=True, env=env_vars).decode("utf-8")
|
||||||
"utf-8"
|
for line in version.splitlines():
|
||||||
)
|
|
||||||
for line in shell_version.splitlines():
|
|
||||||
if "gitVersion" in line:
|
if "gitVersion" in line:
|
||||||
version_line = line.split(":")[1]
|
version_line = line.split(":")[1]
|
||||||
# We identify the commit hash as the string enclosed by double quotation marks.
|
# We identify the commit hash as the string enclosed by double quotation marks.
|
||||||
@ -50,9 +48,7 @@ def get_backports_required_hash_for_shell_version(mongo_shell_path: str | None =
|
|||||||
return commit_hash
|
return commit_hash
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
raise ValueError(
|
raise ValueError(f"Could not find a valid commit hash from the {mongod_path} mongo binary.")
|
||||||
f"Could not find a valid commit hash from the {mongo_shell_path} mongo binary."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_git_file_content(commit_hash: str) -> str:
|
def get_git_file_content(commit_hash: str) -> str:
|
||||||
@ -112,14 +108,12 @@ def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Log
|
|||||||
# mongo shell executable.
|
# mongo shell executable.
|
||||||
from buildscripts.resmokelib import multiversionconstants
|
from buildscripts.resmokelib import multiversionconstants
|
||||||
|
|
||||||
shell_version = {
|
old_mongod = {
|
||||||
MultiversionOptions.LAST_LTS: multiversionconstants.LAST_LTS_MONGO_BINARY,
|
MultiversionOptions.LAST_LTS: multiversionconstants.LAST_LTS_MONGOD_BINARY,
|
||||||
MultiversionOptions.LAST_CONTINUOUS: multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY,
|
MultiversionOptions.LAST_CONTINUOUS: multiversionconstants.LAST_CONTINUOUS_MONGOD_BINARY,
|
||||||
}[old_bin_version]
|
}[old_bin_version]
|
||||||
|
|
||||||
old_version_commit_hash = get_backports_required_hash_for_shell_version(
|
old_version_commit_hash = get_backports_required_hash(old_mongod)
|
||||||
mongo_shell_path=shell_version
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get the yaml contents from the old commit.
|
# Get the yaml contents from the old commit.
|
||||||
logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}")
|
logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}")
|
||||||
|
|||||||
@ -217,7 +217,7 @@ class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):
|
|||||||
|
|
||||||
def mongo_client(
|
def mongo_client(
|
||||||
self, read_preference=pymongo.ReadPreference.PRIMARY, timeout_millis=30000, **kwargs
|
self, read_preference=pymongo.ReadPreference.PRIMARY, timeout_millis=30000, **kwargs
|
||||||
):
|
) -> pymongo.MongoClient:
|
||||||
"""Return a pymongo.MongoClient connecting to this fixture with specified 'read_preference'.
|
"""Return a pymongo.MongoClient connecting to this fixture with specified 'read_preference'.
|
||||||
|
|
||||||
The PyMongo driver will wait up to 'timeout_millis' milliseconds
|
The PyMongo driver will wait up to 'timeout_millis' milliseconds
|
||||||
@ -524,7 +524,9 @@ def create_fixture_table(fixture):
|
|||||||
return "Fixture status:\n" + table
|
return "Fixture status:\n" + table
|
||||||
|
|
||||||
|
|
||||||
def build_client(node, auth_options=None, read_preference=pymongo.ReadPreference.PRIMARY):
|
def build_client(
|
||||||
|
node, auth_options=None, read_preference=pymongo.ReadPreference.PRIMARY, **kwargs
|
||||||
|
) -> pymongo.MongoClient:
|
||||||
"""Authenticate client for the 'authenticationDatabase' and return the client."""
|
"""Authenticate client for the 'authenticationDatabase' and return the client."""
|
||||||
if auth_options is not None:
|
if auth_options is not None:
|
||||||
return node.mongo_client(
|
return node.mongo_client(
|
||||||
@ -533,9 +535,10 @@ def build_client(node, auth_options=None, read_preference=pymongo.ReadPreference
|
|||||||
authSource=auth_options["authenticationDatabase"],
|
authSource=auth_options["authenticationDatabase"],
|
||||||
authMechanism=auth_options["authenticationMechanism"],
|
authMechanism=auth_options["authenticationMechanism"],
|
||||||
read_preference=read_preference,
|
read_preference=read_preference,
|
||||||
|
**kwargs,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
return node.mongo_client(read_preference=read_preference)
|
return node.mongo_client(read_preference=read_preference, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
# Represents a row in a node info table.
|
# Represents a row in a node info table.
|
||||||
|
|||||||
@ -587,3 +587,4 @@ def _add_testing_set_parameters(suite_set_parameters):
|
|||||||
# The placeholder is needed so older versions don't have this option won't have this value set.
|
# The placeholder is needed so older versions don't have this option won't have this value set.
|
||||||
suite_set_parameters.setdefault("backtraceLogFile", True)
|
suite_set_parameters.setdefault("backtraceLogFile", True)
|
||||||
suite_set_parameters.setdefault("disableTransitionFromLatestToLastContinuous", False)
|
suite_set_parameters.setdefault("disableTransitionFromLatestToLastContinuous", False)
|
||||||
|
suite_set_parameters.setdefault("oplogApplicationEnforcesSteadyStateConstraints", True)
|
||||||
|
|||||||
@ -341,7 +341,7 @@ class _AddRemoveShardThread(threading.Thread):
|
|||||||
if err.code == self._ILLEGAL_OPERATION:
|
if err.code == self._ILLEGAL_OPERATION:
|
||||||
if "Can't move an internal resharding collection" in str(err):
|
if "Can't move an internal resharding collection" in str(err):
|
||||||
return True
|
return True
|
||||||
if "Can't reshard a timeseries collection" in str(err):
|
if "Can't register a temporary collection" in str(err):
|
||||||
return True
|
return True
|
||||||
for regex in self._UNMOVABLE_NAMESPACE_REGEXES:
|
for regex in self._UNMOVABLE_NAMESPACE_REGEXES:
|
||||||
if re.search(regex, namespace):
|
if re.search(regex, namespace):
|
||||||
|
|||||||
@ -7,7 +7,7 @@ import sys
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from buildscripts.resmokelib import errors
|
from buildscripts.resmokelib import config, errors
|
||||||
from buildscripts.resmokelib.mongo_fuzzer_configs import generate_normal_mongo_parameters
|
from buildscripts.resmokelib.mongo_fuzzer_configs import generate_normal_mongo_parameters
|
||||||
from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
|
from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
|
||||||
from buildscripts.resmokelib.testing.fixtures import replicaset, shardedcluster, standalone
|
from buildscripts.resmokelib.testing.fixtures import replicaset, shardedcluster, standalone
|
||||||
@ -23,6 +23,13 @@ def validate_runtime_parameter_spec(spec):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_client(node, auth_options):
|
||||||
|
"""Build a pymongo MongoClient for the given node with the given auth options."""
|
||||||
|
if config.IS_ASAN:
|
||||||
|
return fixture_interface.build_client(node, auth_options, timeout_millis=120000)
|
||||||
|
return fixture_interface.build_client(node, auth_options)
|
||||||
|
|
||||||
|
|
||||||
class RuntimeParametersState:
|
class RuntimeParametersState:
|
||||||
"""Encapsulates the runtime-state of a set of parameters we are fuzzing. Tracks the last time we set a parameter value and holds
|
"""Encapsulates the runtime-state of a set of parameters we are fuzzing. Tracks the last time we set a parameter value and holds
|
||||||
the logic for generating new values."""
|
the logic for generating new values."""
|
||||||
@ -222,7 +229,7 @@ class FuzzRuntimeParameters(interface.Hook):
|
|||||||
|
|
||||||
def _invoke_get_parameter_and_log(self, node):
|
def _invoke_get_parameter_and_log(self, node):
|
||||||
"""Helper to print the current state of a node's runtime-fuzzable parameters. Only usable once before_suite has initialized the runtime state of the parameters."""
|
"""Helper to print the current state of a node's runtime-fuzzable parameters. Only usable once before_suite has initialized the runtime state of the parameters."""
|
||||||
client = fixture_interface.build_client(node, self._auth_options)
|
client = build_client(node, self._auth_options)
|
||||||
params_to_get = (
|
params_to_get = (
|
||||||
self._mongos_param_state.get_spec()
|
self._mongos_param_state.get_spec()
|
||||||
if client.is_mongos
|
if client.is_mongos
|
||||||
@ -366,9 +373,7 @@ class _SetParameterThread(threading.Thread):
|
|||||||
mongod_params_to_set,
|
mongod_params_to_set,
|
||||||
)
|
)
|
||||||
for node in repl_set.nodes:
|
for node in repl_set.nodes:
|
||||||
invoke_set_parameter(
|
invoke_set_parameter(build_client(node, self._auth_options), mongod_params_to_set)
|
||||||
fixture_interface.build_client(node, self._auth_options), mongod_params_to_set
|
|
||||||
)
|
|
||||||
|
|
||||||
for standalone in self._standalone_fixtures:
|
for standalone in self._standalone_fixtures:
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
@ -376,9 +381,7 @@ class _SetParameterThread(threading.Thread):
|
|||||||
standalone.port,
|
standalone.port,
|
||||||
mongod_params_to_set,
|
mongod_params_to_set,
|
||||||
)
|
)
|
||||||
invoke_set_parameter(
|
invoke_set_parameter(build_client(standalone, self._auth_options), mongod_params_to_set)
|
||||||
fixture_interface.build_client(standalone, self._auth_options), mongod_params_to_set
|
|
||||||
)
|
|
||||||
|
|
||||||
for mongos in self._mongos_fixtures:
|
for mongos in self._mongos_fixtures:
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
@ -386,6 +389,4 @@ class _SetParameterThread(threading.Thread):
|
|||||||
mongos.port,
|
mongos.port,
|
||||||
mongos_params_to_set,
|
mongos_params_to_set,
|
||||||
)
|
)
|
||||||
invoke_set_parameter(
|
invoke_set_parameter(build_client(mongos, self._auth_options), mongos_params_to_set)
|
||||||
fixture_interface.build_client(mongos, self._auth_options), mongos_params_to_set
|
|
||||||
)
|
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import pymongo.mongo_client
|
|||||||
from pymongo.collection import Collection
|
from pymongo.collection import Collection
|
||||||
from pymongo.database import Database
|
from pymongo.database import Database
|
||||||
|
|
||||||
|
from buildscripts.resmokelib import config
|
||||||
from buildscripts.resmokelib.testing.fixtures.external import ExternalFixture
|
from buildscripts.resmokelib.testing.fixtures.external import ExternalFixture
|
||||||
from buildscripts.resmokelib.testing.fixtures.interface import build_client
|
from buildscripts.resmokelib.testing.fixtures.interface import build_client
|
||||||
from buildscripts.resmokelib.testing.fixtures.standalone import MongoDFixture
|
from buildscripts.resmokelib.testing.fixtures.standalone import MongoDFixture
|
||||||
@ -131,7 +132,12 @@ def validate_node(
|
|||||||
auth_options = None
|
auth_options = None
|
||||||
if shell_options and "authenticationMechanism" in shell_options:
|
if shell_options and "authenticationMechanism" in shell_options:
|
||||||
auth_options = shell_options
|
auth_options = shell_options
|
||||||
client = build_client(node, auth_options, pymongo.ReadPreference.PRIMARY_PREFERRED)
|
if config.IS_ASAN:
|
||||||
|
client = build_client(
|
||||||
|
node, auth_options, pymongo.ReadPreference.PRIMARY_PREFERRED, timeout_millis=120000
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
client = build_client(node, auth_options, pymongo.ReadPreference.PRIMARY_PREFERRED)
|
||||||
|
|
||||||
# Skip validating collections for arbiters.
|
# Skip validating collections for arbiters.
|
||||||
admin_db = client.get_database("admin")
|
admin_db = client.get_database("admin")
|
||||||
|
|||||||
@ -3,6 +3,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
@ -22,6 +23,7 @@ def read_sha_file(filename):
|
|||||||
content = f.read()
|
content = f.read()
|
||||||
return content.strip().split()[0]
|
return content.strip().split()[0]
|
||||||
|
|
||||||
|
|
||||||
def _fetch_remote_sha256_hash(s3_path: str):
|
def _fetch_remote_sha256_hash(s3_path: str):
|
||||||
downloaded = False
|
downloaded = False
|
||||||
result = None
|
result = None
|
||||||
@ -40,7 +42,7 @@ def _fetch_remote_sha256_hash(s3_path: str):
|
|||||||
|
|
||||||
if downloaded:
|
if downloaded:
|
||||||
result = read_sha_file(tempfile_name)
|
result = read_sha_file(tempfile_name)
|
||||||
|
|
||||||
if tempfile_name and os.path.exists(tempfile_name):
|
if tempfile_name and os.path.exists(tempfile_name):
|
||||||
os.unlink(tempfile_name)
|
os.unlink(tempfile_name)
|
||||||
|
|
||||||
@ -61,7 +63,7 @@ def _verify_s3_hash(s3_path: str, local_path: str, expected_hash: str) -> None:
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Hash mismatch for {s3_path}, expected {expected_hash} but got {hash_string}"
|
f"Hash mismatch for {s3_path}, expected {expected_hash} but got {hash_string}"
|
||||||
)
|
)
|
||||||
print(f"File is valid: {local_path} (sha256: {expected_hash})")
|
|
||||||
|
|
||||||
def validate_file(s3_path, output_path, remote_sha_allowed):
|
def validate_file(s3_path, output_path, remote_sha_allowed):
|
||||||
hexdigest = S3_SHA256_HASHES.get(s3_path)
|
hexdigest = S3_SHA256_HASHES.get(s3_path)
|
||||||
@ -69,7 +71,7 @@ def validate_file(s3_path, output_path, remote_sha_allowed):
|
|||||||
print(f"Validating against hard coded sha256: {hexdigest}")
|
print(f"Validating against hard coded sha256: {hexdigest}")
|
||||||
_verify_s3_hash(s3_path, output_path, hexdigest)
|
_verify_s3_hash(s3_path, output_path, hexdigest)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if not remote_sha_allowed:
|
if not remote_sha_allowed:
|
||||||
raise ValueError(f"No SHA256 hash available for {s3_path}")
|
raise ValueError(f"No SHA256 hash available for {s3_path}")
|
||||||
|
|
||||||
@ -82,13 +84,13 @@ def validate_file(s3_path, output_path, remote_sha_allowed):
|
|||||||
print(f"Validating against remote sha256 {hexdigest}\n({s3_path}.sha256)")
|
print(f"Validating against remote sha256 {hexdigest}\n({s3_path}.sha256)")
|
||||||
else:
|
else:
|
||||||
print(f"Failed to download remote sha256 at {s3_path}.sha256)")
|
print(f"Failed to download remote sha256 at {s3_path}.sha256)")
|
||||||
|
|
||||||
if hexdigest:
|
if hexdigest:
|
||||||
_verify_s3_hash(s3_path, output_path, hexdigest)
|
_verify_s3_hash(s3_path, output_path, hexdigest)
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"No SHA256 hash available for {s3_path}")
|
raise ValueError(f"No SHA256 hash available for {s3_path}")
|
||||||
|
|
||||||
|
|
||||||
def _download_and_verify(s3_path, output_path, remote_sha_allowed):
|
def _download_and_verify(s3_path, output_path, remote_sha_allowed):
|
||||||
for i in range(5):
|
for i in range(5):
|
||||||
@ -98,8 +100,9 @@ def _download_and_verify(s3_path, output_path, remote_sha_allowed):
|
|||||||
download_from_s3_with_boto(s3_path, output_path)
|
download_from_s3_with_boto(s3_path, output_path)
|
||||||
except Exception:
|
except Exception:
|
||||||
download_from_s3_with_requests(s3_path, output_path)
|
download_from_s3_with_requests(s3_path, output_path)
|
||||||
|
|
||||||
validate_file(s3_path, output_path, remote_sha_allowed)
|
validate_file(s3_path, output_path, remote_sha_allowed)
|
||||||
|
break
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
print("Download failed:")
|
print("Download failed:")
|
||||||
@ -133,7 +136,15 @@ def download_s3_binary(
|
|||||||
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||||
tempfile_name = temp_file.name
|
tempfile_name = temp_file.name
|
||||||
_download_and_verify(s3_path, tempfile_name, remote_sha_allowed)
|
_download_and_verify(s3_path, tempfile_name, remote_sha_allowed)
|
||||||
|
|
||||||
|
try:
|
||||||
os.replace(tempfile_name, local_path)
|
os.replace(tempfile_name, local_path)
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno == 18: # EXDEV cross filesystem error, need to use a mv
|
||||||
|
shutil.move(tempfile_name, local_path)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
print(f"Downloaded and verified {s3_path} -> {local_path}")
|
print(f"Downloaded and verified {s3_path} -> {local_path}")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@ -146,8 +157,6 @@ def download_s3_binary(
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Download and verify S3 binary.")
|
parser = argparse.ArgumentParser(description="Download and verify S3 binary.")
|
||||||
parser.add_argument("s3_path", help="S3 URL to download from")
|
parser.add_argument("s3_path", help="S3 URL to download from")
|
||||||
parser.add_argument("local_path", nargs="?", help="Optional output file path")
|
parser.add_argument("local_path", nargs="?", help="Optional output file path")
|
||||||
|
|||||||
@ -12,6 +12,7 @@ def compute_sha256(file_path: str) -> str:
|
|||||||
sha256.update(block)
|
sha256.update(block)
|
||||||
return sha256.hexdigest()
|
return sha256.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def write_sha256_file(file_path: str, hash_value: str):
|
def write_sha256_file(file_path: str, hash_value: str):
|
||||||
sha256_path = file_path + ".sha256"
|
sha256_path = file_path + ".sha256"
|
||||||
file_name = os.path.basename(file_path)
|
file_name = os.path.basename(file_path)
|
||||||
@ -19,6 +20,7 @@ def write_sha256_file(file_path: str, hash_value: str):
|
|||||||
f.write(f"{hash_value} {file_name}\n")
|
f.write(f"{hash_value} {file_name}\n")
|
||||||
print(f"Wrote SHA-256 to {sha256_path}")
|
print(f"Wrote SHA-256 to {sha256_path}")
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("Usage: sha256sum.py <file>")
|
print("Usage: sha256sum.py <file>")
|
||||||
@ -32,5 +34,6 @@ def main():
|
|||||||
hash_value = compute_sha256(file_path)
|
hash_value = compute_sha256(file_path)
|
||||||
write_sha256_file(file_path, hash_value)
|
write_sha256_file(file_path, hash_value)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
29
buildscripts/sbom/BUILD.bazel
Normal file
29
buildscripts/sbom/BUILD.bazel
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
load("@rules_python//python:defs.bzl", "py_binary", "py_library")
|
||||||
|
|
||||||
|
py_library(
|
||||||
|
name = "config",
|
||||||
|
srcs = ["config.py"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
py_library(
|
||||||
|
name = "endorctl_utils",
|
||||||
|
srcs = ["endorctl_utils.py"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
py_binary(
|
||||||
|
name = "generate_sbom",
|
||||||
|
srcs = ["generate_sbom.py"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"config",
|
||||||
|
"endorctl_utils",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
py_binary(
|
||||||
|
name = "sbom_files_pr",
|
||||||
|
srcs = ["sbom_files_pr.py"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
5
buildscripts/sbom/OWNERS.yml
Normal file
5
buildscripts/sbom/OWNERS.yml
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
version: 2.0.0
|
||||||
|
filters:
|
||||||
|
- "*":
|
||||||
|
approvers:
|
||||||
|
- 10gen/code-review-team-ssdlc
|
||||||
184
buildscripts/sbom/config.py
Normal file
184
buildscripts/sbom/config.py
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""generate_sbom.py config. Operational configuration values stored separately from the core code."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
logger = logging.getLogger("generate_sbom")
|
||||||
|
logger.setLevel(logging.NOTSET)
|
||||||
|
|
||||||
|
|
||||||
|
# ################ Component Filters ################
|
||||||
|
|
||||||
|
# List of Endor Labs SBOM components that must be removed before processing
|
||||||
|
endor_components_remove = []
|
||||||
|
|
||||||
|
# bom-ref prefixes (Endor Labs has been changing them, so add all that we have seen)
|
||||||
|
prefixes = [
|
||||||
|
"pkg:c/github.com/",
|
||||||
|
"pkg:generic/github.com/",
|
||||||
|
"pkg:github/",
|
||||||
|
]
|
||||||
|
|
||||||
|
components_remove = [
|
||||||
|
# Endor Labs includes the main component in 'components'. This is not standard, so we remove it.
|
||||||
|
"10gen/mongo",
|
||||||
|
# should be pkg:github/antirez/linenoise - waiting on Endor Labs fix
|
||||||
|
"amokhuginnsson/replxx",
|
||||||
|
# a transitive dependency of s2 that is not necessary to include
|
||||||
|
"sparsehash/sparsehash",
|
||||||
|
]
|
||||||
|
|
||||||
|
for component in components_remove:
|
||||||
|
for prefix in prefixes:
|
||||||
|
endor_components_remove.append(prefix + component)
|
||||||
|
|
||||||
|
# ################ Component Renaming ################
|
||||||
|
# Endor does not have syntactically valid PURLs for C/C++ packages.
|
||||||
|
# e.g.,
|
||||||
|
# Invalid: pkg:c/github.com/abseil/abseil-cpp@20250512.1
|
||||||
|
# Valid: pkg:github/abseil/abseil-cpp@20250512.1
|
||||||
|
# Run string replacements to correct for this:
|
||||||
|
endor_components_rename = [
|
||||||
|
["pkg:c/sourceware.org/git/valgrind", "pkg:generic/valgrind/valgrind"],
|
||||||
|
["pkg:generic/sourceware.org/git/valgrind", "pkg:generic/valgrind/valgrind"],
|
||||||
|
["pkg:generic/zlib.net/zlib", "pkg:github/madler/zlib"],
|
||||||
|
["pkg:generic/tartarus.org/libstemmer", "pkg:github/snowballstem/snowball"],
|
||||||
|
["pkg:generic/intel.com/intel-dfp-math", "pkg:generic/intel/IntelRDFPMathLib"],
|
||||||
|
["pkg:c/git.openldap.org/openldap/openldap", "pkg:generic/openldap/openldap"],
|
||||||
|
["pkg:generic/github.com/", "pkg:github/"],
|
||||||
|
["pkg:c/github.com/", "pkg:github/"],
|
||||||
|
]
|
||||||
|
|
||||||
|
# ################ Version Transformation ################
|
||||||
|
|
||||||
|
# In some cases we need to transform the version string to strip out tag-related text
|
||||||
|
# It is unknown what patterns may appear in the future, so we have targeted (not broad) regex
|
||||||
|
# This a list of 'pattern' and 'repl' inputs to re.sub()
|
||||||
|
RE_VER_NUM = r"(0|[1-9]\d*)"
|
||||||
|
RE_VER_LBL = r"(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"
|
||||||
|
RE_SEMVER = rf"{RE_VER_NUM}\.{RE_VER_NUM}\.{RE_VER_NUM}{RE_VER_LBL}"
|
||||||
|
regex_semver = re.compile(RE_SEMVER)
|
||||||
|
|
||||||
|
VERSION_PATTERN_REPL = [
|
||||||
|
# 'debian/1.28.1-1' pkg:github/mongodb/mongo-c-driver (temporary workaround)
|
||||||
|
[re.compile(rf"^debian/({RE_SEMVER})-\d$"), r"\1"],
|
||||||
|
# 'gperftools-2.9.1' pkg:github/gperftools/gperftools
|
||||||
|
# 'mongo/v1.5.2' pkg:github/google/benchmark
|
||||||
|
# 'mongodb-8.2.0-alpha2' pkg:github/wiredtiger/wiredtiger
|
||||||
|
# 'release-1.12.0' pkg:github/apache/avro
|
||||||
|
# 'yaml-cpp-0.6.3' pkg:github/jbeder/yaml-cpp
|
||||||
|
[re.compile(rf"^[-a-z]+[-/][vr]?({RE_SEMVER})$"), r"\1"],
|
||||||
|
# 'asio-1-34-2' pkg:github/chriskohlhoff/asio
|
||||||
|
# 'cares-1_27_0' pkg:github/c-ares/c-ares
|
||||||
|
[
|
||||||
|
re.compile(rf"^[a-z]+-{RE_VER_NUM}[_-]{RE_VER_NUM}[_-]{RE_VER_NUM}{RE_VER_LBL}$"),
|
||||||
|
r"\1.\2.\3",
|
||||||
|
],
|
||||||
|
# 'pcre2-10.40' pkg:github/pcre2project/pcre2
|
||||||
|
[re.compile(rf"^[a-z0-9]+-({RE_VER_NUM}\.{RE_VER_NUM})$"), r"\1"],
|
||||||
|
# 'icu-release-57-1' pkg:github/unicode-org/icu
|
||||||
|
[re.compile(rf"^[a-z]+-?[a-z]+-{RE_VER_NUM}-{RE_VER_NUM}$"), r"\1.\2"],
|
||||||
|
# 'v2.6.0' pkg:github/confluentinc/librdkafka
|
||||||
|
# 'r2.5.1'
|
||||||
|
[re.compile(rf"^[rv]({RE_SEMVER})$"), r"\1"],
|
||||||
|
# 'v2025.04.21.00' pkg:github/facebook/folly
|
||||||
|
[re.compile(r"^v(\d+\.\d+\.\d+\.\d+)$"), r"\1"],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_semver_from_release_version(release_ver: str) -> str:
|
||||||
|
"""Extract the version number from string with tags or other annotations"""
|
||||||
|
if release_ver:
|
||||||
|
for re_obj, repl in VERSION_PATTERN_REPL:
|
||||||
|
if re_obj.match(release_ver):
|
||||||
|
return re_obj.sub(repl, release_ver)
|
||||||
|
return release_ver
|
||||||
|
|
||||||
|
|
||||||
|
# region special component use-case functions
|
||||||
|
|
||||||
|
|
||||||
|
def get_version_from_wiredtiger_release_info(wt_dir: str) -> str:
|
||||||
|
"""Get version from 'RELEASE_INFO' file in the wiredtiger folder"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
v = {}
|
||||||
|
try:
|
||||||
|
for l in open(os.path.join(wt_dir, "RELEASE_INFO"), "r", encoding="utf-8"):
|
||||||
|
if re.match(r"WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=", l):
|
||||||
|
exec(l, v)
|
||||||
|
wt_ver = "%d.%d.%d" % (
|
||||||
|
v["WIREDTIGER_VERSION_MAJOR"],
|
||||||
|
v["WIREDTIGER_VERSION_MINOR"],
|
||||||
|
v["WIREDTIGER_VERSION_PATCH"],
|
||||||
|
)
|
||||||
|
return wt_ver
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error loading file from {wt_dir}")
|
||||||
|
logger.error(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_version_sasl_from_workspace(file_path: str) -> str:
|
||||||
|
"""Determine the version that is pulled for Windows Cyrus SASL by searching WORKSPACE.bazel"""
|
||||||
|
# e.g.,
|
||||||
|
# "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
|
||||||
|
try:
|
||||||
|
with open(file_path, "r") as file:
|
||||||
|
for line in file:
|
||||||
|
if line.strip().startswith(
|
||||||
|
'"https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-'
|
||||||
|
):
|
||||||
|
return line.strip().split("windows_cyrus_sasl-")[1].split(".zip")[0]
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Unable to load {file_path}")
|
||||||
|
logger.warning(e)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def process_component_special_cases(
|
||||||
|
component_key: str, component: dict, versions: dict, repo_root: str
|
||||||
|
) -> None:
|
||||||
|
"""Handle special cases for specific components."""
|
||||||
|
## Special case for Cyrus SASL ##
|
||||||
|
if component_key == "pkg:github/cyrusimap/cyrus-sasl":
|
||||||
|
# Cycrus SASL is optionally loaded as a Windows library, when needed. There is no source code for Endor Labs to scan.
|
||||||
|
# The version of Cyrus SASL that is used is defined in the WORKSPACE.bazel file:
|
||||||
|
# "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
|
||||||
|
# Rather than add the complexity of Bazel queries to this script, we just search the text.
|
||||||
|
|
||||||
|
versions["import_script"] = get_version_sasl_from_workspace(repo_root + "/WORKSPACE.bazel")
|
||||||
|
logger.info(
|
||||||
|
f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'WORKSPACE.bazel' file"
|
||||||
|
)
|
||||||
|
|
||||||
|
## Special case for wiredtiger ##
|
||||||
|
elif component_key == "pkg:github/wiredtiger/wiredtiger":
|
||||||
|
# MongoDB release branches import wiredtiger commits via a bot. These commits will likely not line up with a release or tag.
|
||||||
|
# Endor labs will try to pull the nearest release/tag, but we want the more precise commit hash, which is stored in:
|
||||||
|
# src/third_party/wiredtiget/import.data
|
||||||
|
occurrences = component.get("evidence", {}).get("occurrences", [])
|
||||||
|
if occurrences:
|
||||||
|
location = occurrences[0].get("location")
|
||||||
|
versions["import_script"] = get_version_from_wiredtiger_release_info(
|
||||||
|
f"{repo_root}/{location}"
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'RELEASE_INFO' file"
|
||||||
|
)
|
||||||
|
|
||||||
|
## Special case for opentelemetry-cpp ##
|
||||||
|
elif component_key == "pkg:github/open-telemetry/opentelemetry-cpp":
|
||||||
|
# The opentelementry import script has the mongodb-forks version ref in Major.Minor format (e.g., 1.17), which deviates from
|
||||||
|
# what the open-telemetry/opentelemetry-cpp project uses (v{SEMVER}). This corrects the version string by adding a '.0', if needed
|
||||||
|
if re.match(rf"^{RE_VER_NUM}\.{RE_VER_NUM}$", versions["import_script"]):
|
||||||
|
versions["import_script"] += ".0"
|
||||||
|
logger.info(
|
||||||
|
f"VERSION SPECIAL CASE: {component_key}: Adjusted import script version string to semver format: '{versions['import_script']}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# endregion special component use-case functions
|
||||||
486
buildscripts/sbom/endorctl_utils.py
Normal file
486
buildscripts/sbom/endorctl_utils.py
Normal file
@ -0,0 +1,486 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Utility functions for the Endor Labs API via endorctl
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
logger = logging.getLogger("generate_sbom")
|
||||||
|
logger.setLevel(logging.NOTSET)
|
||||||
|
|
||||||
|
default_field_masks = {
|
||||||
|
"PackageVersion": [
|
||||||
|
"context",
|
||||||
|
"meta",
|
||||||
|
"processing_status",
|
||||||
|
"spec.package_name",
|
||||||
|
"spec.resolved_dependencies.dependencies",
|
||||||
|
"spec.source_code_reference",
|
||||||
|
],
|
||||||
|
"ScanResult": [
|
||||||
|
"context",
|
||||||
|
"meta",
|
||||||
|
"spec.end_time",
|
||||||
|
"spec.logs",
|
||||||
|
"spec.refs",
|
||||||
|
"spec.start_time",
|
||||||
|
"spec.status",
|
||||||
|
"spec.versions",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_default_field_mask(kind):
|
||||||
|
default_field_mask = default_field_masks.get(kind, [])
|
||||||
|
return ",".join(default_field_mask)
|
||||||
|
|
||||||
|
|
||||||
|
class EndorResourceKind(Enum):
|
||||||
|
"""Enumeration for Endor Labs API resource kinds"""
|
||||||
|
|
||||||
|
PROJECT = "Project"
|
||||||
|
REPOSITORY_VERSION = "RepositoryVersion"
|
||||||
|
SCAN_RESULT = "ScanResult"
|
||||||
|
PACKAGE_VERSION = "PackageVersion"
|
||||||
|
|
||||||
|
|
||||||
|
class EndorContextType(Enum):
|
||||||
|
"""Most objects include a common nested object called Context. Contexts keep objects from different scans separated.
|
||||||
|
https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/common-fields/#context"""
|
||||||
|
|
||||||
|
# Objects from a scan of the default branch. All objects in the OSS namespace are in the main context. The context ID is always default.
|
||||||
|
MAIN = "CONTEXT_TYPE_MAIN"
|
||||||
|
# Objects from a scan of a specific branch. The context ID is the branch reference name.
|
||||||
|
REF = "CONTEXT_TYPE_REF"
|
||||||
|
# Objects from a PR scan. The context ID is the PR UUID. Objects in this context are deleted after 30 days.
|
||||||
|
CI_RUN = "CONTEXT_TYPE_CI_RUN"
|
||||||
|
|
||||||
|
|
||||||
|
class EndorFilter:
|
||||||
|
"""Provide standard filters for Endor Labs API resource kinds"""
|
||||||
|
|
||||||
|
def __init__(self, context_id=None, context_type=None):
|
||||||
|
self.context_id = context_id
|
||||||
|
self.context_type = context_type
|
||||||
|
|
||||||
|
def _base_filters(self):
|
||||||
|
base_filters = []
|
||||||
|
if self.context_id:
|
||||||
|
base_filters.append(f"context.id=={self.context_id}")
|
||||||
|
if self.context_type:
|
||||||
|
base_filters.append(f"context.type=={self.context_type}")
|
||||||
|
|
||||||
|
return base_filters
|
||||||
|
|
||||||
|
def repository_version(self, project_uuid=None, sha=None, ref=None):
|
||||||
|
filters = self._base_filters()
|
||||||
|
if project_uuid:
|
||||||
|
filters.append(f"meta.parent_uuid=={project_uuid}")
|
||||||
|
if sha:
|
||||||
|
filters.append(f"spec.version.sha=={sha}")
|
||||||
|
if ref:
|
||||||
|
filters.append(f"spec.version.ref=={ref}")
|
||||||
|
|
||||||
|
return " and ".join(filters)
|
||||||
|
|
||||||
|
def package_version(
|
||||||
|
self,
|
||||||
|
context_type: EndorContextType = None,
|
||||||
|
context_id=None,
|
||||||
|
project_uuid=None,
|
||||||
|
name=None,
|
||||||
|
package_name=None,
|
||||||
|
):
|
||||||
|
filters = self._base_filters()
|
||||||
|
if context_type:
|
||||||
|
filters.append(f"context.type=={context_type.value}")
|
||||||
|
if context_type:
|
||||||
|
filters.append(f"context.id=={context_id}")
|
||||||
|
if project_uuid:
|
||||||
|
filters.append(f"spec.project_uuid=={project_uuid}")
|
||||||
|
if name:
|
||||||
|
filters.append(f"spec.package_name=={name}")
|
||||||
|
if package_name:
|
||||||
|
filters.append(f"meta.name=={package_name}")
|
||||||
|
|
||||||
|
return " and ".join(filters)
|
||||||
|
|
||||||
|
def scan_result(
|
||||||
|
self,
|
||||||
|
context_type: EndorContextType = None,
|
||||||
|
project_uuid=None,
|
||||||
|
ref=None,
|
||||||
|
sha=None,
|
||||||
|
status=None,
|
||||||
|
):
|
||||||
|
filters = self._base_filters()
|
||||||
|
if context_type:
|
||||||
|
filters.append(f"context.type=={context_type.value}")
|
||||||
|
if project_uuid:
|
||||||
|
filters.append(f"meta.parent_uuid=={project_uuid}")
|
||||||
|
if ref:
|
||||||
|
filters.append(f"spec.versions.ref contains '{ref}'")
|
||||||
|
if sha:
|
||||||
|
filters.append(f"spec.versions.sha contains '{sha}'")
|
||||||
|
if status:
|
||||||
|
filters.append(f"spec.status=={status}")
|
||||||
|
|
||||||
|
return " and ".join(filters)
|
||||||
|
|
||||||
|
|
||||||
|
class EndorCtl:
|
||||||
|
"""Interact with endorctl (Endor Labs CLI)"""
|
||||||
|
|
||||||
|
# region internal functions
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
namespace,
|
||||||
|
retry_limit=5,
|
||||||
|
sleep_duration=30,
|
||||||
|
endorctl_path="endorctl",
|
||||||
|
config_path=None,
|
||||||
|
):
|
||||||
|
self.namespace = namespace
|
||||||
|
self.retry_limit = retry_limit
|
||||||
|
self.sleep_duration = sleep_duration
|
||||||
|
self.endorctl_path = endorctl_path
|
||||||
|
self.config_path = config_path
|
||||||
|
|
||||||
|
def _call_endorctl(self, command, subcommand, **kwargs):
|
||||||
|
"""https://docs.endorlabs.com/endorctl/"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
command = [self.endorctl_path, command, subcommand, f"--namespace={self.namespace}"]
|
||||||
|
if self.config_path:
|
||||||
|
command.append(f"--config-path={self.config_path}")
|
||||||
|
|
||||||
|
# parse args into flags
|
||||||
|
for key, value in kwargs.items():
|
||||||
|
# Handle endorctl flags with hyphens that are defined in the script with underscores
|
||||||
|
flag = key.replace("_", "-")
|
||||||
|
if value:
|
||||||
|
command.append(f"--{flag}={value}")
|
||||||
|
logger.info("Running: %s", " ".join(command))
|
||||||
|
|
||||||
|
result = subprocess.run(command, capture_output=True, text=True, check=True)
|
||||||
|
|
||||||
|
resource = json.loads(result.stdout)
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
logger.error(f"Error executing command: {e}")
|
||||||
|
logger.error(e.stderr)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logger.error(f"Error decoding JSON: {e}")
|
||||||
|
logger.error(f"Stdout: {result.stdout}")
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
logger.error(f"FileNotFoundError: {e}")
|
||||||
|
logger.error(
|
||||||
|
f"'endorctl' not found in path '{self.endorctl_path}'. Supply the correct path, run 'buildscripts/install_endorctl.sh' or visit https://docs.endorlabs.com/endorctl/install-and-configure/"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"An unexpected error occurred: {e}")
|
||||||
|
else:
|
||||||
|
return resource
|
||||||
|
|
||||||
|
def _api_get(self, resource, **kwargs):
|
||||||
|
"""https://docs.endorlabs.com/endorctl/commands/api/"""
|
||||||
|
return self._call_endorctl("api", "get", resource=resource, **kwargs)
|
||||||
|
|
||||||
|
def _api_list(self, resource, filter=None, retry=True, **kwargs):
|
||||||
|
"""https://docs.endorlabs.com/endorctl/commands/api/"""
|
||||||
|
# If this script is run immediately after making a commit, Endor Labs will likely not yet have created the assocaited ScanResult object. The wait/retry logic below handles this scenario.
|
||||||
|
tries = 0
|
||||||
|
while True:
|
||||||
|
tries += 1
|
||||||
|
result = self._call_endorctl("api", "list", resource=resource, filter=filter, **kwargs)
|
||||||
|
|
||||||
|
# The expected output of 'endorctl api list' is: { "list": { "objects": [...] } }
|
||||||
|
# We want to just return the objects. In case we get an empty list, return a list
|
||||||
|
# with a single None to avoid having to handle index errors downstream.
|
||||||
|
if result and result["list"].get("objects") and len(result["list"]["objects"]) > 0:
|
||||||
|
return result["list"]["objects"]
|
||||||
|
elif retry:
|
||||||
|
logger.info(
|
||||||
|
f"API LIST: Resource not found: {resource} with filter '{filter}' in namespace '{self.namespace}'"
|
||||||
|
)
|
||||||
|
if tries <= self.retry_limit:
|
||||||
|
logger.info(
|
||||||
|
f"API LIST: Waiting for {self.sleep_duration} seconds before retry attempt {tries} of {self.retry_limit}"
|
||||||
|
)
|
||||||
|
time.sleep(self.sleep_duration)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
f"API LIST: Maximum number of allowed retries {self.retry_limit} attempted with no {resource} found using filter '{filter}'"
|
||||||
|
)
|
||||||
|
return [None]
|
||||||
|
else:
|
||||||
|
return [None]
|
||||||
|
|
||||||
|
def _check_resource(self, resource, resource_description) -> None:
|
||||||
|
if not resource:
|
||||||
|
raise LookupError(f"Resource not found: {resource_description}")
|
||||||
|
logger.info(f"Retrieved: {resource_description}")
|
||||||
|
|
||||||
|
# endregion internal functions
|
||||||
|
|
||||||
|
# region resource functions
|
||||||
|
def get_resource(self, resource, uuid=None, name=None, field_mask=None, **kwargs):
|
||||||
|
"""https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/"""
|
||||||
|
if not field_mask:
|
||||||
|
field_mask = _get_default_field_mask(resource)
|
||||||
|
return self._api_get(
|
||||||
|
resource=resource, uuid=uuid, name=name, field_mask=field_mask, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_resources(
|
||||||
|
self,
|
||||||
|
resource,
|
||||||
|
filter=None,
|
||||||
|
field_mask=None,
|
||||||
|
sort_path="meta.create_time",
|
||||||
|
sort_order="descending",
|
||||||
|
retry=True,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
"""https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/"""
|
||||||
|
if not field_mask:
|
||||||
|
field_mask = _get_default_field_mask(resource)
|
||||||
|
return self._api_list(
|
||||||
|
resource=resource,
|
||||||
|
filter=filter,
|
||||||
|
field_mask=field_mask,
|
||||||
|
sort_path=sort_path,
|
||||||
|
sort_order=sort_order,
|
||||||
|
retry=retry,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_project(self, git_url):
|
||||||
|
resource_kind = EndorResourceKind.PROJECT.value
|
||||||
|
resource_description = (
|
||||||
|
f"{resource_kind} with name '{git_url}' in namespace '{self.namespace}'"
|
||||||
|
)
|
||||||
|
project = self.get_resource(resource_kind, name=git_url)
|
||||||
|
self._check_resource(project, resource_description)
|
||||||
|
return project
|
||||||
|
|
||||||
|
def get_repository_version(self, filter=None, retry=True):
|
||||||
|
resource_kind = EndorResourceKind.REPOSITORY_VERSION.value
|
||||||
|
resource_description = (
|
||||||
|
f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'"
|
||||||
|
)
|
||||||
|
repository_version = self.get_resources(
|
||||||
|
resource_kind, filter=filter, retry=retry, page_size=1
|
||||||
|
)[0]
|
||||||
|
self._check_resource(repository_version, resource_description)
|
||||||
|
return repository_version
|
||||||
|
|
||||||
|
def get_scan_result(self, filter=None, retry=True):
|
||||||
|
resource_kind = EndorResourceKind.SCAN_RESULT.value
|
||||||
|
resource_description = (
|
||||||
|
f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'"
|
||||||
|
)
|
||||||
|
scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0]
|
||||||
|
self._check_resource(scan_result, resource_description)
|
||||||
|
uuid = scan_result.get("uuid")
|
||||||
|
start_time = scan_result["spec"].get("start_time")
|
||||||
|
refs = scan_result["spec"].get("refs")
|
||||||
|
polling_start_time = datetime.now()
|
||||||
|
while True:
|
||||||
|
status = scan_result["spec"].get("status")
|
||||||
|
end_time = scan_result["spec"].get("end_time")
|
||||||
|
if status == "STATUS_SUCCESS":
|
||||||
|
logger.info(
|
||||||
|
f" Scan completed successfully. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}."
|
||||||
|
)
|
||||||
|
return scan_result
|
||||||
|
elif status == "STATUS_RUNNING":
|
||||||
|
logger.info(
|
||||||
|
f" Scan is running. ScanResult uuid {uuid} for refs {refs} started at {start_time}."
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f" Waiting {self.sleep_duration} seconds before checking status. Total wait time: {(datetime.now() - polling_start_time).total_seconds()/60:.2f} minutes"
|
||||||
|
)
|
||||||
|
time.sleep(self.sleep_duration)
|
||||||
|
scan_result = self.get_resources(
|
||||||
|
resource_kind, filter=filter, retry=retry, page_size=1
|
||||||
|
)[0]
|
||||||
|
elif status == "STATUS_PARTIAL_SUCCESS":
|
||||||
|
scan_logs = scan_result["spec"].get("logs")
|
||||||
|
raise RuntimeError(
|
||||||
|
f" Scan completed, but with critical warnings or errors. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}"
|
||||||
|
)
|
||||||
|
elif status == "STATUS_FAILURE":
|
||||||
|
scan_logs = scan_result["spec"].get("logs")
|
||||||
|
raise RuntimeError(
|
||||||
|
f" Scan failed. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_package_versions(self, filter):
|
||||||
|
resource_kind = EndorResourceKind.PACKAGE_VERSION.value
|
||||||
|
resource_description = (
|
||||||
|
f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'"
|
||||||
|
)
|
||||||
|
package_versions = self.get_resources(resource_kind, filter=filter)
|
||||||
|
self._check_resource(package_versions, resource_description)
|
||||||
|
return package_versions
|
||||||
|
|
||||||
|
def export_sbom(
|
||||||
|
self,
|
||||||
|
package_version_uuid=None,
|
||||||
|
package_version_uuids=None,
|
||||||
|
package_version_name=None,
|
||||||
|
app_name=None,
|
||||||
|
project_name=None,
|
||||||
|
project_uuid=None,
|
||||||
|
):
|
||||||
|
"""Export an SBOM from Endor Labs
|
||||||
|
|
||||||
|
Valid parameter sets (other combinations result in an error from 'endorctl'):
|
||||||
|
Single-Package SBOM:
|
||||||
|
package_version_uuid
|
||||||
|
package_version_name
|
||||||
|
Multi-Package SBOM:
|
||||||
|
package_version_uuids,app_name
|
||||||
|
project_uuid,app_name,app_name
|
||||||
|
project_name,app_name,app_name
|
||||||
|
|
||||||
|
https://docs.endorlabs.com/endorctl/commands/sbom/export/
|
||||||
|
"""
|
||||||
|
if package_version_uuids:
|
||||||
|
package_version_uuids = ",".join(package_version_uuids)
|
||||||
|
return self._call_endorctl(
|
||||||
|
"sbom",
|
||||||
|
"export",
|
||||||
|
package_version_uuid=package_version_uuid,
|
||||||
|
package_version_uuids=package_version_uuids,
|
||||||
|
package_version_name=package_version_name,
|
||||||
|
app_name=app_name,
|
||||||
|
project_name=project_name,
|
||||||
|
project_uuid=project_uuid,
|
||||||
|
)
|
||||||
|
|
||||||
|
# endregion resource functions
|
||||||
|
|
||||||
|
# region workflow functions
|
||||||
|
def get_sbom_for_commit(self, git_url: str, commit_sha: str) -> dict:
|
||||||
|
"""Export SBOM for the PR commit (sha)"""
|
||||||
|
|
||||||
|
endor_filter = EndorFilter()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Project: get uuid
|
||||||
|
project = self.get_project(git_url)
|
||||||
|
project_uuid = project["uuid"]
|
||||||
|
app_name = project["spec"]["git"]["full_name"]
|
||||||
|
|
||||||
|
# RepositoryVersion: get the context for the PR scan
|
||||||
|
endor_filter.context_type = EndorContextType.CI_RUN.value
|
||||||
|
filter_str = endor_filter.repository_version(project_uuid, commit_sha)
|
||||||
|
repository_version = self.get_repository_version(filter_str)
|
||||||
|
context_id = repository_version["context"]["id"]
|
||||||
|
|
||||||
|
# ScanResult: wait for a completed scan
|
||||||
|
endor_filter.context_id = context_id
|
||||||
|
filter_str = endor_filter.scan_result(project_uuid)
|
||||||
|
self.get_scan_result(filter_str)
|
||||||
|
|
||||||
|
# PackageVersions: get package versions for SBOM
|
||||||
|
filter_str = endor_filter.package_version(project_uuid)
|
||||||
|
package_versions = self.get_package_versions(filter_str)
|
||||||
|
package_version_uuids = [
|
||||||
|
package_version["uuid"] for package_version in package_versions
|
||||||
|
]
|
||||||
|
package_version_names = [
|
||||||
|
package_version["meta"]["name"] for package_version in package_versions
|
||||||
|
]
|
||||||
|
|
||||||
|
# Export SBOM
|
||||||
|
sbom = self.export_sbom(package_version_uuids=package_version_uuids, app_name=app_name)
|
||||||
|
print(
|
||||||
|
f"Retrieved: CycloneDX SBOM for PackageVersion(s), name: {package_version_names}, uuid: {package_version_uuids}"
|
||||||
|
)
|
||||||
|
return sbom
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Exception: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
def get_sbom_for_branch(self, git_url: str, branch: str) -> dict:
|
||||||
|
"""Export lastest SBOM for a monitored branch/ref"""
|
||||||
|
|
||||||
|
endor_filter = EndorFilter()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Project: get uuid
|
||||||
|
project = self.get_project(git_url)
|
||||||
|
project_uuid = project["uuid"]
|
||||||
|
app_name = project["spec"]["git"]["full_name"]
|
||||||
|
|
||||||
|
# RepositoryVersion: get the context for the latest branch scan
|
||||||
|
filter_str = endor_filter.repository_version(project_uuid, ref=branch)
|
||||||
|
repository_version = self.get_repository_version(filter_str)
|
||||||
|
repository_version_uuid = repository_version["uuid"]
|
||||||
|
repository_version_ref = repository_version["spec"]["version"]["ref"]
|
||||||
|
repository_version_sha = repository_version["spec"]["version"]["sha"]
|
||||||
|
repository_version_scan_object_status = repository_version["scan_object"]["status"]
|
||||||
|
if repository_version_scan_object_status != "STATUS_SCANNED":
|
||||||
|
logger.warning(
|
||||||
|
f"RepositoryVersion (uuid: {repository_version_uuid}, ref: {repository_version_ref}, sha: {repository_version_sha}) scan status is '{repository_version_scan_object_status}' (expected 'STATUS_SCANNED')"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ScanResult: search for a completed scan
|
||||||
|
filter_str = endor_filter.scan_result(
|
||||||
|
EndorContextType.MAIN, project_uuid, repository_version_ref, repository_version_sha
|
||||||
|
)
|
||||||
|
scan_result = self.get_scan_result(filter_str, retry=False)
|
||||||
|
project_uuid = scan_result["meta"]["parent_uuid"]
|
||||||
|
|
||||||
|
# PackageVersions: get package versions for SBOM
|
||||||
|
if branch == "master":
|
||||||
|
context_type = EndorContextType.MAIN
|
||||||
|
context_id = "default"
|
||||||
|
else:
|
||||||
|
context_type = EndorContextType.REF
|
||||||
|
context_id = branch
|
||||||
|
filter_str = endor_filter.package_version(context_type, context_id, project_uuid)
|
||||||
|
package_version = self.get_package_versions(filter_str)[0]
|
||||||
|
package_version_name = package_version["meta"]["name"]
|
||||||
|
package_version_uuid = package_version["uuid"]
|
||||||
|
|
||||||
|
# Export SBOM
|
||||||
|
sbom = self.export_sbom(package_version_uuid=package_version_uuid, app_name=app_name)
|
||||||
|
logger.info(
|
||||||
|
f"SBOM: Retrieved CycloneDX SBOM for PackageVersion, name: {package_version_name}, uuid {package_version_uuid}"
|
||||||
|
)
|
||||||
|
return sbom
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Exception: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
def get_sbom_for_project(self, git_url: str) -> dict:
|
||||||
|
"""Export latest SBOM for EndorCtl project default branch"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Project: get uuid
|
||||||
|
project = self.get_project(git_url)
|
||||||
|
project_uuid = project["uuid"]
|
||||||
|
app_name = project["spec"]["git"]["full_name"]
|
||||||
|
|
||||||
|
# Export SBOM
|
||||||
|
sbom = self.export_sbom(project_uuid=project_uuid, app_name=app_name)
|
||||||
|
logger.info(f"Retrieved: CycloneDX SBOM for Project {app_name}")
|
||||||
|
return sbom
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Exception: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# endregion workflow functions
|
||||||
1016
buildscripts/sbom/generate_sbom.py
Executable file
1016
buildscripts/sbom/generate_sbom.py
Executable file
File diff suppressed because it is too large
Load Diff
199
buildscripts/sbom/install_endorctl.sh
Executable file
199
buildscripts/sbom/install_endorctl.sh
Executable file
@ -0,0 +1,199 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -o errexit
|
||||||
|
|
||||||
|
echo "+----------------------------------------------------------------------------+"
|
||||||
|
echo "| Script to install the Endor Labs CLI and verify authentication |"
|
||||||
|
echo "| endorctl (https://docs.endorlabs.com/endorctl/) |"
|
||||||
|
echo "| Environment Variables (optional): |"
|
||||||
|
echo "| ENDOR_INSTALL_PATH - only if in CI or not installed with homebrew or npm |"
|
||||||
|
echo "| ENDOR_CONFIG_PATH - endor config directory (default: ~/.endorctl) |"
|
||||||
|
echo "+----------------------------------------------------------------------------+"
|
||||||
|
echo
|
||||||
|
|
||||||
|
function endorctl_check_install() {
|
||||||
|
# Check if installed
|
||||||
|
ENDOR_INSTALLED_PATH=$(command -v endorctl)
|
||||||
|
if [[ -n "$ENDOR_INSTALLED_PATH" ]]; then
|
||||||
|
# Is Installed
|
||||||
|
echo "Binary 'endorctl' is installed in '${ENDOR_INSTALLED_PATH}'."
|
||||||
|
chmod +x $ENDOR_INSTALLED_PATH
|
||||||
|
if [[ -x "$ENDOR_INSTALLED_PATH" ]]; then
|
||||||
|
echo "Binary 'endorctl' is executable."
|
||||||
|
return 0 # True (success)
|
||||||
|
else
|
||||||
|
echo "Binary 'endorctl' is NOT executable after attempting to make it executable."
|
||||||
|
return 1 # False (failure)
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "Binary 'endorctl' is NOT installed or not in PATH."
|
||||||
|
return 1 # False (failure)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function endorctl_install() {
|
||||||
|
|
||||||
|
# Skip trying homebrew and npm if runing in CI
|
||||||
|
if [[ "$CI" == "true" ]]; then
|
||||||
|
echo "---------------------------------"
|
||||||
|
echo "Detected that script is running in CI. Skipping Homebrew and NPM."
|
||||||
|
else
|
||||||
|
# Try brew
|
||||||
|
echo "---------------------------------"
|
||||||
|
echo "Checking if Homebrew is available"
|
||||||
|
if command -v brew --version &>/dev/null; then
|
||||||
|
echo "Attempting to install with Homebrew"
|
||||||
|
brew tap endorlabs/tap
|
||||||
|
brew install endorctl
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "Warning: Homebrew installation failed."
|
||||||
|
else
|
||||||
|
echo "Installed with Homebrew"
|
||||||
|
return 0 # True (success)
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "Homebrew is not available"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Try NPM
|
||||||
|
echo "---------------------------------"
|
||||||
|
echo "Checking if npm is available"
|
||||||
|
if command -v npm --version &>/dev/null; then
|
||||||
|
# Install binary for linux or macos
|
||||||
|
echo "Attempting to install with npm"
|
||||||
|
npm install --global endorctl
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "Warning: npm installation failed."
|
||||||
|
else
|
||||||
|
echo "Installed with npm"
|
||||||
|
return 0 # True (success)
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "npm is not available"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Try binary installation
|
||||||
|
echo "---------------------------------"
|
||||||
|
echo "Attempting binary install"
|
||||||
|
|
||||||
|
if [[ -z "$ENDOR_INSTALL_PATH" ]]; then
|
||||||
|
ENDOR_INSTALL_PATH="${HOME}/.local/bin"
|
||||||
|
fi
|
||||||
|
echo "Installation path set to $ENDOR_INSTALL_PATH"
|
||||||
|
mkdir -p "$ENDOR_INSTALL_PATH"
|
||||||
|
export PATH="${ENDOR_INSTALL_PATH}:$PATH"
|
||||||
|
ENDOR_BIN_PATH="${ENDOR_INSTALL_PATH}/endorctl"
|
||||||
|
|
||||||
|
case $(uname -m) in
|
||||||
|
"x86_64" | "amd64")
|
||||||
|
ARCH="amd64"
|
||||||
|
;;
|
||||||
|
"aarch64" | "arm64")
|
||||||
|
ARCH="arm64"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Error: Unexpected architecture: $(uname -m). Expected x86_64, amd64, or arm64."
|
||||||
|
return 1 # False (failure)
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
case "$OSTYPE" in
|
||||||
|
linux*)
|
||||||
|
PLATFORM="linux"
|
||||||
|
;;
|
||||||
|
darwin*)
|
||||||
|
PLATFORM="macos"
|
||||||
|
;;
|
||||||
|
msys* | cygwin* | "Windows_NT")
|
||||||
|
echo "Error: Automated installation on Windows without npm is not implemented in this script."
|
||||||
|
echo "For manual Windows installation, follow instructions at:"
|
||||||
|
echo " https://docs.endorlabs.com/endorctl/install-and-configure/#download-and-install-the-endorctl-binary-directly"
|
||||||
|
echo ""
|
||||||
|
echo_auth_instructions
|
||||||
|
return 1 # False (failure)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Error: Unexpected OS type: $OSTYPE"
|
||||||
|
return 1 # False (failure)
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
## Download the latest CLI for supported platform and architecture
|
||||||
|
URL="https://api.endorlabs.com/download/latest/endorctl_${PLATFORM}_${ARCH}"
|
||||||
|
echo "Downloading latest CLI for $PLATFORM $ARCH to $BIN_PATH from $URL"
|
||||||
|
curl --silent $URL --output "$ENDOR_BIN_PATH"
|
||||||
|
## Verify the checksum of the binary
|
||||||
|
echo "Verifying checksum of binary"
|
||||||
|
case "$PLATFORM" in
|
||||||
|
linux)
|
||||||
|
echo "$(curl -s https://api.endorlabs.com/sha/latest/endorctl_${PLATFORM}_${ARCH})" $ENDOR_BIN_PATH | sha256sum -c
|
||||||
|
;;
|
||||||
|
macos)
|
||||||
|
echo "$(curl -s https://api.endorlabs.com/sha/latest/endorctl_${PLATFORM}_${ARCH})" $ENDOR_BIN_PATH | shasum -a 256 -c
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
## Modify the permissions of the binary to ensure it is executable
|
||||||
|
echo " Modifying binary permissions to executable"
|
||||||
|
chmod +x $ENDOR_BIN_PATH
|
||||||
|
## Create an alias endorctl of the binary to ensure it is available in other directory
|
||||||
|
alias endorctl=$ENDOR_BIN_PATH
|
||||||
|
|
||||||
|
echo "endorctl installed in $ENDOR_BIN_PATH"
|
||||||
|
return 0 # True (success)
|
||||||
|
}
|
||||||
|
|
||||||
|
function endorctl_check_auth() {
|
||||||
|
# Check authentication
|
||||||
|
echo "Checking authentication with command: endorctl api get --resource Project --namespace mongodb.10gen --name https://github.com/10gen/mongo.git --config-path $ENDOR_CONFIG_PATH"
|
||||||
|
endorctl api get --resource Project --namespace mongodb.10gen --name "https://github.com/10gen/mongo.git" --config-path $ENDOR_CONFIG_PATH >/dev/null
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "Authentication confirmed."
|
||||||
|
return 0 # True (success)
|
||||||
|
else
|
||||||
|
echo "Authentication failure. Command exit code: $?"
|
||||||
|
echo_auth_instructions
|
||||||
|
return 1 # False (failure)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function echo_auth_instructions() {
|
||||||
|
echo ""
|
||||||
|
echo "------------------------------------------------ AUTOMATED AUTH ------------------------------------------------"
|
||||||
|
echo "Set the following environment variables:"
|
||||||
|
echo " export ENDOR_API_CREDENTIALS_KEY=<api-key>"
|
||||||
|
echo " export ENDOR_API_CREDENTIALS_SECRET=<api-key-secret>"
|
||||||
|
echo " export ENDOR_NAMESPACE=mongodb.{github_org}"
|
||||||
|
echo ""
|
||||||
|
echo "--------------------------------------------------- USER AUTH ---------------------------------------------------"
|
||||||
|
echo "To authenticate endorctl, visit the following URL, authenticate via Okta SSO, and copy the authentication token."
|
||||||
|
echo " https://api.endorlabs.com/v1/auth/sso?tenant=mongodb.10gen&redirect=headless"
|
||||||
|
echo "Then run:"
|
||||||
|
echo " endorctl auth --token [AUTH_TOKEN]"
|
||||||
|
echo ""
|
||||||
|
echo "Alternatively, run the init command. Must use headless mode when no GUI is available:"
|
||||||
|
echo " endorctl init --auth-mode=sso --auth-tenant=mongodb.10gen --headless-mode"
|
||||||
|
echo ""
|
||||||
|
echo "Enter 'y' if prompted to overwrite existing configuration and/or delete account keys."
|
||||||
|
echo ""
|
||||||
|
echo "If authentication fails, confirm in MANA that you are a member of a '10gen-endor-labs-*' Okta group."
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# Set/Create config folder
|
||||||
|
if [[ -z "$ENDOR_CONFIG_PATH" ]]; then
|
||||||
|
ENDOR_CONFIG_PATH="${HOME}/.endorctl"
|
||||||
|
fi
|
||||||
|
echo "Config path set to ${ENDOR_CONFIG_PATH}"
|
||||||
|
|
||||||
|
if ! endorctl_check_install; then
|
||||||
|
if ! endorctl_install; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! endorctl_check_auth; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
2625
buildscripts/sbom/metadata.cdx.json
Normal file
2625
buildscripts/sbom/metadata.cdx.json
Normal file
File diff suppressed because it is too large
Load Diff
199
buildscripts/sbom/sbom_files_pr.py
Normal file
199
buildscripts/sbom/sbom_files_pr.py
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Script that opens a PR using a bot to update SBOM-related files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from github import (
|
||||||
|
Commit,
|
||||||
|
GithubException,
|
||||||
|
GithubIntegration,
|
||||||
|
GitRef,
|
||||||
|
PullRequest,
|
||||||
|
Repository,
|
||||||
|
)
|
||||||
|
|
||||||
|
SBOM_FILES = ["sbom.json", "README.third_party.md"]
|
||||||
|
|
||||||
|
|
||||||
|
def get_repository(github_owner, github_repo, app_id, _private_key) -> Repository.Repository:
|
||||||
|
"""
|
||||||
|
Gets the mongo github repository
|
||||||
|
"""
|
||||||
|
app = GithubIntegration(int(app_id), _private_key)
|
||||||
|
installation = app.get_repo_installation(github_owner, github_repo)
|
||||||
|
g = installation.get_github_for_installation()
|
||||||
|
return g.get_repo(f"{github_owner}/{github_repo}")
|
||||||
|
|
||||||
|
|
||||||
|
def get_pull_request(branch_gitref: GitRef.GitRef) -> PullRequest.PullRequest | None:
|
||||||
|
"""
|
||||||
|
Gets the pull request for the branch ref, if it exists
|
||||||
|
"""
|
||||||
|
pulls = branch_gitref
|
||||||
|
print("get_pull_request:")
|
||||||
|
for pull in pulls:
|
||||||
|
print(" pull: ", pull)
|
||||||
|
if pulls.totalCount > 0:
|
||||||
|
pull = pulls[0]
|
||||||
|
print(f"Found open PR #{pull.number} '{pull.title}'")
|
||||||
|
return pull
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def create_branch(base_branch, new_branch) -> None:
|
||||||
|
"""
|
||||||
|
Create a new branch or get existing branch.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"Attempting to create branch '{new_branch}' with base branch '{base_branch}'.")
|
||||||
|
ref = f"refs/heads/{new_branch}"
|
||||||
|
base_repo_branch = repo.get_branch(base_branch)
|
||||||
|
sha = base_repo_branch.commit.sha
|
||||||
|
repo.create_git_ref(ref=ref, sha=sha)
|
||||||
|
print(f"Created branch '{new_branch}', ref: {ref}, sha: {sha}")
|
||||||
|
except GithubException as e:
|
||||||
|
if e.status == 422:
|
||||||
|
print(f"Branch {new_branch} already exists, ref: {ref}")
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def read_text_file(file_path: str) -> str:
|
||||||
|
"""Read a text file and return as string"""
|
||||||
|
try:
|
||||||
|
with open(file_path, "r", encoding="utf-8") as file:
|
||||||
|
content = file.read()
|
||||||
|
return content
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(f"ERROR: The file '{file_path}' was not found.")
|
||||||
|
return f"ERROR: The file '{file_path}' was not found."
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An error occurred: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="This script checks for changes to SBOM and related files and creats a PR if files have been updated.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--github-owner", help="GitHub org/owner (e.g., 10gen).", type=str)
|
||||||
|
parser.add_argument("--github-repo", help="GitHub repository name (e.g., mongo).", type=str)
|
||||||
|
parser.add_argument("--base-branch", help="base branch to merge into.", type=str)
|
||||||
|
parser.add_argument("--new-branch", help="New branch for the PR.", type=str)
|
||||||
|
parser.add_argument("--pr-title", help="Title for the PR.", type=str)
|
||||||
|
parser.add_argument(
|
||||||
|
"--saved-warnings", help="Path to file to include as text in PR message.", type=str
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--app-id",
|
||||||
|
help="GitHub App ID used for authentication.",
|
||||||
|
type=str,
|
||||||
|
default=os.getenv("MONGO_PR_BOT_APP_ID"),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--private-key",
|
||||||
|
help="Key to use for GitHub App authentication.",
|
||||||
|
type=str,
|
||||||
|
default=os.getenv("MONGO_PR_BOT_PRIVATE_KEY"),
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if not args.app_id or not args.private_key:
|
||||||
|
parser.error(
|
||||||
|
"Must define --app-id or env MONGO_PR_BOT_APP_ID and --private-key or env MONGO_PR_BOT_PRIVATE_KEY."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Replace spaces with newline, if applicable
|
||||||
|
private_key = (
|
||||||
|
args.private_key[:31] + args.private_key[31:-29].replace(" ", "\n") + args.private_key[-29:]
|
||||||
|
)
|
||||||
|
|
||||||
|
repo = get_repository(args.github_owner, args.github_repo, args.app_id, private_key)
|
||||||
|
print("repo: ", repo)
|
||||||
|
|
||||||
|
HAS_UPDATE = False
|
||||||
|
|
||||||
|
for file_path in SBOM_FILES:
|
||||||
|
original_file = repo.get_contents(file_path, ref=f"refs/heads/{args.base_branch}")
|
||||||
|
print("original_file: ", original_file)
|
||||||
|
original_content = original_file.decoded_content.decode()
|
||||||
|
try:
|
||||||
|
with open(file_path, "r", encoding="utf-8") as file:
|
||||||
|
new_content = file.read()
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("Error: file '%s' not found.", file_path)
|
||||||
|
|
||||||
|
# Compare content with removed Endor Labs version to avoid triggering a new SBOM on only that change
|
||||||
|
PATTERN = r'{"name":"EndorLabsInc","version":".*"}'
|
||||||
|
REPL = r'{"name":"EndorLabsInc","version":""}'
|
||||||
|
original_content_compare = re.sub(PATTERN, REPL, "".join(original_content.split()))
|
||||||
|
new_content_compare = re.sub(PATTERN, REPL, "".join(new_content.split()))
|
||||||
|
|
||||||
|
if original_content_compare != new_content_compare:
|
||||||
|
create_branch(args.base_branch, args.new_branch)
|
||||||
|
original_file_new_branch = repo.get_contents(
|
||||||
|
file_path, ref=f"refs/heads/{args.new_branch}"
|
||||||
|
)
|
||||||
|
print("original_file_new_branch: ", original_file_new_branch)
|
||||||
|
|
||||||
|
print("New file is different from original file.")
|
||||||
|
print("repo.update_file:")
|
||||||
|
print(f" message: Updating '{file_path}'")
|
||||||
|
print(" path: ", file_path)
|
||||||
|
print(" sha: ", original_file_new_branch.sha)
|
||||||
|
print(" content:")
|
||||||
|
print(new_content[:128])
|
||||||
|
print("...[truncated]...")
|
||||||
|
print(new_content[-128:])
|
||||||
|
print(" branch: ", args.new_branch)
|
||||||
|
time.sleep(10) # Wait to reduce chance of 409 errors
|
||||||
|
update_file_result = repo.update_file(
|
||||||
|
message=f"Updating '{file_path}'",
|
||||||
|
path=file_path,
|
||||||
|
sha=original_file_new_branch.sha,
|
||||||
|
content=new_content,
|
||||||
|
branch=args.new_branch,
|
||||||
|
)
|
||||||
|
print("update_file_result: ", update_file_result)
|
||||||
|
commit: Commit = update_file_result.get("commit")
|
||||||
|
print("commit: ", commit)
|
||||||
|
|
||||||
|
HAS_UPDATE = True
|
||||||
|
|
||||||
|
if HAS_UPDATE:
|
||||||
|
# Get open PR or create new PR
|
||||||
|
pull_requests = repo.get_pulls(
|
||||||
|
state="open", head=f"{args.github_owner}:{args.new_branch}", base=args.base_branch
|
||||||
|
)
|
||||||
|
if pull_requests.totalCount:
|
||||||
|
pull_request = pull_requests[0]
|
||||||
|
print("pull_request: ", pull_request)
|
||||||
|
else:
|
||||||
|
pr_body = "Automated PR updating SBOM and related files."
|
||||||
|
print("Creating PR:")
|
||||||
|
print(f" title={args.pr_title}")
|
||||||
|
print(f" head={args.new_branch}")
|
||||||
|
print(f" base={args.base_branch}")
|
||||||
|
print(f" body={pr_body}")
|
||||||
|
|
||||||
|
pull_request = repo.create_pull(
|
||||||
|
title=args.pr_title,
|
||||||
|
head=args.new_branch,
|
||||||
|
base=args.base_branch,
|
||||||
|
body=pr_body,
|
||||||
|
)
|
||||||
|
print("pull_request: ", pull_request)
|
||||||
|
|
||||||
|
if args.saved_warnings:
|
||||||
|
pr_comment = "The following warnings were output by the SBOM generation script:\n"
|
||||||
|
if os.path.isfile(args.saved_warnings):
|
||||||
|
pr_comment += read_text_file(args.saved_warnings)
|
||||||
|
comment = pull_request.create_issue_comment(pr_comment)
|
||||||
|
print("Added PR comment: ", comment)
|
||||||
|
else:
|
||||||
|
print(f"Files '{SBOM_FILES}' have not changed. Skipping PR.")
|
||||||
@ -4,9 +4,15 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
import jsonschema
|
from license_expression import get_spdx_licensing
|
||||||
from referencing import Registry, Resource
|
from referencing import Registry, Resource
|
||||||
|
|
||||||
|
try:
|
||||||
|
import jsonschema
|
||||||
|
except ImportError:
|
||||||
|
print("'jsonschema' not found. Continuing without it.")
|
||||||
|
jsonschema = None
|
||||||
|
|
||||||
BOM_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "bom-1.5.schema.json")
|
BOM_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "bom-1.5.schema.json")
|
||||||
SPDX_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "spdx.schema.json")
|
SPDX_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "spdx.schema.json")
|
||||||
SPDX_SCHEMA_REF = "spdx.schema.json"
|
SPDX_SCHEMA_REF = "spdx.schema.json"
|
||||||
@ -32,8 +38,9 @@ MISSING_TEAM_ERROR = "Component must include a 'internal:team_responsible' prope
|
|||||||
SCHEMA_MATCH_FAILURE = "File did not match the CycloneDX schema"
|
SCHEMA_MATCH_FAILURE = "File did not match the CycloneDX schema"
|
||||||
MISSING_VERSION_IN_SBOM_COMPONENT_ERROR = "Component must include a version."
|
MISSING_VERSION_IN_SBOM_COMPONENT_ERROR = "Component must include a version."
|
||||||
MISSING_VERSION_IN_IMPORT_FILE_ERROR = "Missing version in the import file: "
|
MISSING_VERSION_IN_IMPORT_FILE_ERROR = "Missing version in the import file: "
|
||||||
|
MISSING_LICENSE_IN_SBOM_COMPONENT_ERROR = "Component must include a license."
|
||||||
COULD_NOT_FIND_OR_READ_SCRIPT_FILE_ERROR = "Could not find or read the import script file"
|
COULD_NOT_FIND_OR_READ_SCRIPT_FILE_ERROR = "Could not find or read the import script file"
|
||||||
VERSION_MISMATCH_ERROR = "Version mismatch: "
|
VERSION_MISMATCH_ERROR = "Version mismatch (may simply be an artifact of SBOM automation): "
|
||||||
|
|
||||||
|
|
||||||
# A class for managing error messages for components
|
# A class for managing error messages for components
|
||||||
@ -115,30 +122,49 @@ def strip_extra_prefixes(string_with_prefix: str) -> str:
|
|||||||
return string_with_prefix.removeprefix("mongo/").removeprefix("v")
|
return string_with_prefix.removeprefix("mongo/").removeprefix("v")
|
||||||
|
|
||||||
|
|
||||||
def validate_evidence(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
|
def validate_license(component: dict, error_manager: ErrorManager) -> None:
|
||||||
if "evidence" not in component or "occurrences" not in component["evidence"]:
|
if "licenses" not in component:
|
||||||
error_manager.append_full_error_message(MISSING_EVIDENCE_ERROR)
|
error_manager.append_full_error_message(MISSING_LICENSE_IN_SBOM_COMPONENT_ERROR)
|
||||||
return
|
return
|
||||||
|
|
||||||
occurrences = component["evidence"]["occurrences"]
|
valid_license = False
|
||||||
if not occurrences:
|
expression = None
|
||||||
error_manager.append_full_error_message(
|
for component_license in component["licenses"]:
|
||||||
"'evidence.occurrences' field must include at least one location."
|
if "expression" in component_license:
|
||||||
)
|
expression = component_license.get("expression")
|
||||||
for occurrence in occurrences:
|
elif "license" in component_license:
|
||||||
location = occurrence["location"]
|
if "id" in component_license["license"]:
|
||||||
|
# Should be a valid SPDX license ID
|
||||||
|
expression = component_license["license"].get("id")
|
||||||
|
elif "name" in component_license["license"]:
|
||||||
|
# If SPDX does not define the license used, the name field may be used to provide the license name
|
||||||
|
valid_license = True
|
||||||
|
|
||||||
if not os.path.exists(location) and not SKIP_FILE_CHECKING:
|
if not valid_license:
|
||||||
error_manager.append_full_error_message("location does not exist in repo.")
|
licensing_validate = get_spdx_licensing().validate(expression, validate=True)
|
||||||
|
# ExpressionInfo(
|
||||||
|
# original_expression='',
|
||||||
|
# normalized_expression='',
|
||||||
|
# errors=[],
|
||||||
|
# invalid_symbols=[]
|
||||||
|
# )
|
||||||
|
valid_license = not licensing_validate.errors or not licensing_validate.invalid_symbols
|
||||||
|
if not valid_license:
|
||||||
|
error_manager.append_full_error_message(licensing_validate)
|
||||||
|
return
|
||||||
|
|
||||||
if location.startswith(THIRD_PARTY_LOCATION_PREFIX):
|
|
||||||
lib = location.removeprefix(THIRD_PARTY_LOCATION_PREFIX)
|
def validate_evidence(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
|
||||||
if lib in third_party_libs:
|
if component.get("scope") == "required":
|
||||||
third_party_libs.remove(lib)
|
if "evidence" not in component or "occurrences" not in component["evidence"]:
|
||||||
|
error_manager.append_full_error_message(MISSING_EVIDENCE_ERROR)
|
||||||
|
return
|
||||||
|
|
||||||
|
validate_location(component, third_party_libs, error_manager)
|
||||||
|
|
||||||
|
|
||||||
def validate_properties(component: dict, error_manager: ErrorManager) -> None:
|
def validate_properties(component: dict, error_manager: ErrorManager) -> None:
|
||||||
has_team_responsible_property = False
|
has_team_responsible_property = False or component.get("scope") == "excluded"
|
||||||
script_path = ""
|
script_path = ""
|
||||||
if "properties" in component:
|
if "properties" in component:
|
||||||
for prop in component["properties"]:
|
for prop in component["properties"]:
|
||||||
@ -146,43 +172,62 @@ def validate_properties(component: dict, error_manager: ErrorManager) -> None:
|
|||||||
has_team_responsible_property = True
|
has_team_responsible_property = True
|
||||||
elif prop["name"] == "import_script_path":
|
elif prop["name"] == "import_script_path":
|
||||||
script_path = prop["value"]
|
script_path = prop["value"]
|
||||||
|
|
||||||
if not has_team_responsible_property:
|
if not has_team_responsible_property:
|
||||||
error_manager.append_full_error_message(MISSING_TEAM_ERROR)
|
error_manager.append_full_error_message(MISSING_TEAM_ERROR)
|
||||||
|
|
||||||
|
if script_path:
|
||||||
|
script_path_is_file = os.path.isfile(script_path)
|
||||||
|
if not script_path_is_file:
|
||||||
|
error_manager.append_full_error_message(COULD_NOT_FIND_OR_READ_SCRIPT_FILE_ERROR)
|
||||||
|
# Only look for VERSION if the import script is a shell script file
|
||||||
|
elif script_path.endswith(".sh"):
|
||||||
|
script_version = get_script_version(script_path, "VERSION", error_manager)
|
||||||
|
if script_version == "":
|
||||||
|
error_manager.append_full_error_message(
|
||||||
|
MISSING_VERSION_IN_IMPORT_FILE_ERROR + script_path
|
||||||
|
)
|
||||||
|
|
||||||
if not component.get("version"):
|
if not component.get("version"):
|
||||||
error_manager.append_full_error_message(MISSING_VERSION_IN_SBOM_COMPONENT_ERROR)
|
error_manager.append_full_error_message(MISSING_VERSION_IN_SBOM_COMPONENT_ERROR)
|
||||||
return
|
return
|
||||||
|
|
||||||
comp_version = component["version"]
|
|
||||||
# If the version is unknown or the script path property is absent, the version
|
|
||||||
# check is not possible (these are valid options and no error is generated).
|
|
||||||
if comp_version == "Unknown" or script_path == "":
|
|
||||||
return
|
|
||||||
|
|
||||||
# At this point a version is attempted to be read from the import script file
|
|
||||||
script_version = get_script_version(script_path, "VERSION", error_manager)
|
|
||||||
if script_version == "":
|
|
||||||
error_manager.append_full_error_message(MISSING_VERSION_IN_IMPORT_FILE_ERROR + script_path)
|
|
||||||
elif strip_extra_prefixes(script_version) != strip_extra_prefixes(comp_version):
|
|
||||||
error_manager.append_full_error_message(
|
|
||||||
VERSION_MISMATCH_ERROR
|
|
||||||
+ f"\nscript version:{script_version}\nsbom version:{comp_version}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_component(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
|
def validate_component(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
|
||||||
error_manager.update_component_attribute(component["name"])
|
error_manager.update_component_attribute(component["name"])
|
||||||
if "scope" not in component:
|
if "scope" not in component:
|
||||||
error_manager.append_full_error_message("component must include a scope.")
|
error_manager.append_full_error_message("component must include a scope.")
|
||||||
elif component["scope"] != "optional":
|
else:
|
||||||
validate_evidence(component, third_party_libs, error_manager)
|
validate_evidence(component, third_party_libs, error_manager)
|
||||||
validate_properties(component, error_manager)
|
validate_properties(component, error_manager)
|
||||||
|
validate_license(component, error_manager)
|
||||||
|
|
||||||
if "purl" not in component and "cpe" not in component:
|
if "purl" not in component and "cpe" not in component:
|
||||||
error_manager.append_full_error_message(MISSING_PURL_CPE_ERROR)
|
error_manager.append_full_error_message(MISSING_PURL_CPE_ERROR)
|
||||||
error_manager.update_component_attribute("")
|
error_manager.update_component_attribute("")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_location(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
|
||||||
|
if "evidence" in component:
|
||||||
|
if "occurrences" not in component["evidence"]:
|
||||||
|
error_manager.append_full_error_message(
|
||||||
|
"'evidence.occurrences' field must include at least one location."
|
||||||
|
)
|
||||||
|
|
||||||
|
occurrences = component["evidence"]["occurrences"]
|
||||||
|
for occurrence in occurrences:
|
||||||
|
if "location" in occurrence:
|
||||||
|
location = occurrence["location"]
|
||||||
|
|
||||||
|
if not os.path.exists(location) and not SKIP_FILE_CHECKING:
|
||||||
|
error_manager.append_full_error_message("location does not exist in repo.")
|
||||||
|
|
||||||
|
if location.startswith(THIRD_PARTY_LOCATION_PREFIX):
|
||||||
|
lib = location.removeprefix(THIRD_PARTY_LOCATION_PREFIX)
|
||||||
|
if lib in third_party_libs:
|
||||||
|
third_party_libs.remove(lib)
|
||||||
|
|
||||||
|
|
||||||
def lint_sbom(
|
def lint_sbom(
|
||||||
input_file: str, output_file: str, third_party_libs: set, should_format: bool
|
input_file: str, output_file: str, third_party_libs: set, should_format: bool
|
||||||
) -> ErrorManager:
|
) -> ErrorManager:
|
||||||
@ -197,15 +242,16 @@ def lint_sbom(
|
|||||||
error_manager.append(f"Failed to parse {input_file}: {str(ex)}")
|
error_manager.append(f"Failed to parse {input_file}: {str(ex)}")
|
||||||
return error_manager
|
return error_manager
|
||||||
|
|
||||||
try:
|
if jsonschema:
|
||||||
schema = get_schema()
|
try:
|
||||||
jsonschema.validators.validator_for(schema)(
|
schema = get_schema()
|
||||||
schema, registry=local_schema_registry()
|
jsonschema.validators.validator_for(schema)(
|
||||||
).validate(sbom)
|
schema, registry=local_schema_registry()
|
||||||
except jsonschema.ValidationError as error:
|
).validate(sbom)
|
||||||
error_manager.append(f"{SCHEMA_MATCH_FAILURE} {input_file}")
|
except jsonschema.ValidationError as error:
|
||||||
error_manager.append(error.message)
|
error_manager.append(f"{SCHEMA_MATCH_FAILURE} {input_file}")
|
||||||
return error_manager
|
error_manager.append(error.message)
|
||||||
|
return error_manager
|
||||||
|
|
||||||
components = sbom["components"]
|
components = sbom["components"]
|
||||||
for component in components:
|
for component in components:
|
||||||
@ -257,10 +303,8 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
# the only files in this dir that are not third party libs
|
# the only files in this dir that are not third party libs
|
||||||
third_party_libs.remove("scripts")
|
third_party_libs.remove("scripts")
|
||||||
# wiredtiger will not be included in the sbom since it is considered part of the server
|
# Nothing in this directory is included in Community/EA
|
||||||
third_party_libs.remove("wiredtiger")
|
third_party_libs.remove("private")
|
||||||
# the only files in the sasl dir are BUILD files to setup the sasl library in Windows
|
|
||||||
third_party_libs.remove("sasl")
|
|
||||||
error_manager = lint_sbom(input_file, output_file, third_party_libs, should_format)
|
error_manager = lint_sbom(input_file, output_file, third_party_libs, should_format)
|
||||||
error_manager.print_errors()
|
error_manager.print_errors()
|
||||||
|
|
||||||
|
|||||||
@ -53,7 +53,6 @@ suites:
|
|||||||
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_server.js
|
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_server.js
|
||||||
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_no_server.js
|
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_no_server.js
|
||||||
- jstests/noPassthrough/validate/validate_empty_collection.js
|
- jstests/noPassthrough/validate/validate_empty_collection.js
|
||||||
- jstests/noPassthrough/validate/skip_geo_hash_checks.js
|
|
||||||
- jstests/noPassthrough/validate/validate_memory_limit.js
|
- jstests/noPassthrough/validate/validate_memory_limit.js
|
||||||
- jstests/noPassthrough/validate/validate_with_long_index_name.js
|
- jstests/noPassthrough/validate/validate_with_long_index_name.js
|
||||||
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_block_fcv_change.js
|
- src/mongo/db/modules/enterprise/jstests/live_restore/live_restore_block_fcv_change.js
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user