SERVER-126021 Fix handling of duplicate fields in flat_bson (#53258)

GitOrigin-RevId: ef888a6df901622004aeb4b7362e007b8c4299da
This commit is contained in:
henrikedin 2026-05-06 00:22:30 -04:00 committed by MongoDB Bot
parent 2884bcf912
commit e55c99fa49
17 changed files with 53 additions and 13340 deletions

11
.github/CODEOWNERS vendored
View File

@ -1,11 +0,0 @@
# All backports must be approved by the server release team
* @10gen/server-release
# Exclude some test files and READMEs from the backport approvals
/etc/backports_required_for_multiversion_tests.yml
/etc/*.suppressions
/README.md
**/README.md
# Exclude all the tests under "jstests" directories from backport approvals
**/jstests/**/*

View File

@ -1,33 +0,0 @@
# This configuration is for migrating code from one Git repository to another using Copybara.
# It selectively copies content, excluding specific paths and preserving authorship.
# To test locally
# sourceUrl = "/path/to/source"
# destinationUrl = "/path/to/dest"
sourceUrl = "https://github.com/10gen/mongo.git"
destinationUrl = "https://github.com/mongodb/mongo.git"
core.workflow(
name = "default",
origin = git.origin(
url = sourceUrl,
ref = "v6.0",
),
destination = git.destination(
url = destinationUrl,
fetch = "v6.0",
push = "v6.0",
),
# Change path to the folder you want to publish publicly
origin_files = glob(["**"], exclude = ["src/mongo/db/modules/**"]),
authoring = authoring.pass_thru("MongoDB <mongodb@mongodb.com>"),
mode = "ITERATIVE",
transformations = [
# (^.*?) - matches the first line (without the newline char)
# \n - matches the first newline (or nothing at all if there is no newline). If there is no match then nothing happens
# ((\n|.)*) - matches everything after
# Overall, this copies only the first line of the commit rather than the body
metadata.scrubber("(^.*?)\n((\n|.)*)", replacement = "$1"),
],
)

View File

@ -1,31 +0,0 @@
# This configuration is for migrating code from one Git repository to another using Copybara.
# It selectively copies content, excluding specific paths and preserving authorship.
sourceUrl = "https://github.com/10gen/mongo.git"
destinationUrl = "https://github.com/10gen/mongo-copybara.git"
core.workflow(
name = "default",
origin = git.origin(
url = sourceUrl,
ref = "v6.0",
),
destination = git.destination(
url = destinationUrl,
fetch = "v6.0",
push = "v6.0",
),
# Change path to the folder you want to publish publicly
origin_files = glob(["**"], exclude=["src/mongo/db/modules/**"]),
authoring = authoring.pass_thru("MongoDB <mongodb@mongodb.com>"),
mode = "ITERATIVE",
# Change the path here to the folder you want to publish publicly
transformations = [
# (^.*?) - matches the first line (without the newline char)
# \n - matches the first newline (or nothing at all if there is no newline). If there is no match then nothing happens
# ((\n|.)*) - matches everything after
# Overall, this copies only the first line of the commit rather than the body
metadata.scrubber("(^.*?)\n((\n|.)*)", replacement = "$1"),
],
)

View File

@ -15,7 +15,7 @@ for details on how it works.
In order to generate a task, we typically create a placeholder task. By convention the name of
these tasks should end in "_gen". Most of the time, generated tasks should inherit the
[gen_task_template](https://github.com/10gen/mongo/blob/31864e3866ce9cc54c08463019846ded2ad9e6e5/etc/evergreen_yml_components/definitions.yml#L99-L107)
[gen_task_template](https://github.com/mongodb/mongo/blob/31864e3866ce9cc54c08463019846ded2ad9e6e5/etc/evergreen_yml_components/definitions.yml#L99-L107)
which configures the required dependencies.
The placeholder tasks needs to have the "generate resmoke tasks" function as one of its `commands`.

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,80 +0,0 @@
# Build variants corresponding to Atlas's OS.
buildvariants:
- name: enterprise-rhel-70-64-bit
display_name: "Enterprise RHEL 7.0"
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel70-small
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug
push_path: linux
push_bucket: downloads.10gen.com
push_name: linux
push_arch: x86_64-enterprise-rhel70
compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
multiversion_platform: rhel70
multiversion_edition: enterprise
has_packages: true
packager_script: packager_enterprise.py
packager_arch: x86_64
packager_distro: rhel70
repo_edition: enterprise
scons_cache_scope: shared
test_flags: --excludeWithAnyTags=requires_latch_analyzer
tasks:
- name: compile_test_and_package_serial_TG
distros:
- rhel70-large
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- rhel70-large
- name: compile_build_tools_next_TG
distros:
- rhel70-large
- name: .aggfuzzer
- name: audit
- name: auth_audit_gen
- name: auth_gen
- name: causally_consistent_jscore_txns_passthrough
- name: config_fuzzer_concurrency
- name: config_fuzzer_simulate_crash_concurrency_replication
distros:
- rhel70-large
- name: config_fuzzer_concurrency_replication
distros:
- rhel70-large
- name: config_fuzzer_jsCore
- name: config_fuzzer_replica_sets_jscore_passthrough
distros:
- rhel70-large
- name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
- name: external_auth
- name: external_auth_aws
- name: .jscore .common !.decimal !.sharding
- name: jsCore_txns_large_txns_format
- name: .jstestfuzz .common
- name: libunwind_tests
- name: .logical_session_cache .one_sec
- name: .ocsp
- name: replica_sets_auth_gen
- name: replica_sets_jscore_passthrough
- name: .replica_sets .multi_oplog
- name: sasl
- name: search
- name: search_auth
- name: search_pinned_connections_auth
- name: search_ssl
- name: sharding_auth_audit_gen
- name: sharding_auth_gen
- name: snmp
- name: .crypt
- name: .publish_crypt
- name: unittest_shell_hang_analyzer_gen
- name: vector_search
- name: vector_search_auth
- name: vector_search_ssl
- name: selinux_rhel7_enterprise
- name: .publish
- name: generate_buildid_to_debug_symbols_mapping

View File

@ -1,216 +0,0 @@
# IBM build variants.
buildvariants:
- name: enterprise-rhel-81-ppc64le
display_name: Enterprise RHEL 8.9 PPC64LE
run_on:
- rhel8-power-small
cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
# We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down
compile_flags: --ssl MONGO_DISTMOD=rhel81 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
resmoke_jobs_factor: 0.25
has_packages: true
packager_script: packager_enterprise.py
packager_arch: ppc64le
packager_distro: rhel81
push_path: linux
push_bucket: downloads.10gen.com
push_name: linux
push_arch: ppc64le-enterprise-rhel81
repo_edition: enterprise
multiversion_platform: rhel81
multiversion_architecture: ppc64le
multiversion_edition: enterprise
test_flags: >-
--excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible
timeout_secs: 7200 # 2 hours idle timeout
exec_timeout_secs: 43200 # 12 hours exec timeout
tasks:
- name: compile_test_and_package_serial_TG
distros:
- rhel8-power-large
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- rhel8-power-large
- name: compile_integration_and_test_no_audit_parallel_stream_TG
distros:
- rhel8-power-large
- name: .aggregation .common
- name: audit
- name: .auth !.multiversion !.jscore
- name: .misc_js
- name: .encrypt
- name: .jscore .common !.auth
- name: .read_write_concern
- name: replica_sets_gen
- name: .replica_sets .common
- name: sasl
- name: .sharding .jscore !.wo_snapshot !.multi_stmt
- name: .sharding .common !.multiversion
- name: snmp
- name: .crypt
- name: .publish_crypt
- name: .publish
- name: generate_buildid_to_debug_symbols_mapping
- name: enterprise-rhel-71-ppc64le
display_name: Enterprise RHEL 7.9 PPC64LE
run_on:
- rhel7-power-small
cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
# We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down
compile_flags: --ssl MONGO_DISTMOD=rhel71 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
resmoke_jobs_factor: 0.25
has_packages: true
packager_script: packager_enterprise.py
packager_arch: ppc64le
packager_distro: rhel71
push_path: linux
push_bucket: downloads.10gen.com
push_name: linux
push_arch: ppc64le-enterprise-rhel71
repo_edition: enterprise
multiversion_platform: rhel71
multiversion_architecture: ppc64le
multiversion_edition: enterprise
test_flags: >-
--excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible
timeout_secs: 7200 # 2 hours idle timeout
exec_timeout_secs: 43200 # 12 hours exec timeout
tasks:
- name: compile_test_and_package_serial_TG
distros:
- rhel7-power-large
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- rhel7-power-large
- name: compile_integration_and_test_no_audit_parallel_stream_TG
distros:
- rhel7-power-large
- name: .aggregation .common
- name: audit
- name: .auth !.multiversion !.jscore
- name: .misc_js
- name: .encrypt
- name: .jscore .common !.auth
- name: .read_write_concern
- name: replica_sets_gen
- name: .replica_sets .common
- name: sasl
- name: .sharding .jscore !.wo_snapshot !.multi_stmt
- name: .sharding .common !.multiversion
- name: snmp
- name: .crypt
- name: .publish_crypt
- name: .publish
- name: generate_buildid_to_debug_symbols_mapping
- name: enterprise-rhel-72-s390x
display_name: Enterprise RHEL 7.9 s390x
run_on:
- rhel7-zseries-small
cron: "0 4 * * 0"
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
release_buid: true
compile_flags: --ssl MONGO_DISTMOD=rhel72 -j3 --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
resmoke_jobs_max: 2
has_packages: true
packager_script: packager_enterprise.py
packager_arch: s390x
packager_distro: rhel72
push_path: linux
push_bucket: downloads.10gen.com
push_name: linux
push_arch: s390x-enterprise-rhel72
repo_edition: enterprise
multiversion_platform: rhel72
multiversion_architecture: s390x
multiversion_edition: enterprise
test_flags: >-
--excludeWithAnyTags=incompatible_with_s390x,requires_increased_memlock_limits,requires_latch_analyzer
tasks:
- name: small_compile_test_and_package_serial_TG
distros:
- rhel7-zseries-large
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- rhel7-zseries-large
- name: compile_integration_and_test_no_audit_parallel_stream_TG
distros:
- rhel7-zseries-large
- name: .aggregation .common
- name: audit
- name: .auth !.multiversion !.jscore
- name: .misc_js
- name: .encrypt
- name: .jscore .common !.auth
- name: .read_write_concern
- name: replica_sets_gen
- name: .replica_sets .common
- name: sasl
- name: .sharding .jscore !.wo_snapshot !.multi_stmt
- name: .sharding .common !.multiversion
- name: snmp
- name: .crypt
- name: .publish_crypt
- name: .publish
- name: generate_buildid_to_debug_symbols_mapping
- name: enterprise-rhel-83-s390x
display_name: Enterprise RHEL 8.3 s390x
run_on:
- rhel8-zseries-large-legacy
cron: "0 4 * * 0"
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
release_buid: true
compile_flags: --ssl MONGO_DISTMOD=rhel83 -j16 --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique"
resmoke_jobs_max: 2
has_packages: true
packager_script: packager_enterprise.py
packager_arch: s390x
packager_distro: rhel83
push_path: linux
push_bucket: downloads.10gen.com
push_name: linux
push_arch: s390x-enterprise-rhel83
repo_edition: enterprise
multiversion_platform: rhel83
multiversion_architecture: s390x
multiversion_edition: enterprise
exec_timeout_secs: 86400 # 24 hours exec timeout
test_flags: >-
--excludeWithAnyTags=incompatible_with_s390x,requires_increased_memlock_limits,requires_latch_analyzer
tasks:
- name: compile_test_and_package_serial_TG
- name: compile_integration_and_test_no_audit_parallel_stream_TG
- name: .aggregation .common
- name: audit
- name: .auth !.multiversion !.jscore
- name: .misc_js
- name: .encrypt
- name: .jscore .common !.auth
- name: .read_write_concern
- name: replica_sets_gen
- name: .replica_sets .common
- name: sasl
- name: .sharding .jscore !.wo_snapshot !.multi_stmt
- name: .sharding .common !.multiversion
- name: snmp
- name: .crypt
- name: .publish_crypt
- name: .publish
- name: generate_buildid_to_debug_symbols_mapping

View File

@ -1,115 +0,0 @@
# InMemory build variants that need to be run on LTS but not Rapid releases.
buildvariants:
- name: enterprise-rhel-8-64-bit-inmem
display_name: Enterprise RHEL 8 (inMemory)
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel80-small
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling
compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on
multiversion_platform: rhel80
multiversion_platform_50_or_later: rhel8
multiversion_edition: enterprise
scons_cache_scope: shared
large_distro_name: rhel80-large
tasks:
- name: compile_test_and_package_serial_TG
distros:
- rhel80-xlarge
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- rhel80-xlarge
- name: compile_integration_and_test_no_audit_parallel_stream_TG
distros:
- rhel80-xlarge
- name: .aggfuzzer .common
- name: .aggregation !.unwind !.encrypt
- name: audit
- name: .auth !.multiversion
- name: .causally_consistent !.wo_snapshot !.durable_history
- name: .change_streams !.secondary_reads
- name: .change_stream_fuzzer
- name: .misc_js
- name: .concurrency !.ubsan !.no_txns !.debug_only !.kill_terminate
distros:
- rhel80-medium # Some workloads require a lot of memory, use a bigger machine for this suite.
- name: initial_sync_fuzzer_gen
- name: .jscore .common !.decimal
- name: jsCore_txns_large_txns_format
- name: .jstestfuzz !.initsync
- name: .logical_session_cache
- name: .multi_shard .common
- name: multi_stmt_txn_jscore_passthrough_with_migration_gen
- name: .read_write_concern !.durable_history
- name: replica_sets_gen
- name: .replica_sets .common
- name: .replica_sets .multi_oplog
- name: replica_sets_multi_stmt_txn_jscore_passthrough
- name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen
distros:
- rhel80-medium
- name: .resharding_fuzzer
- name: .retry
- name: rollback_fuzzer_gen
- name: sasl
- name: secondary_reads_passthrough_gen
- name: session_jscore_passthrough
- name: sharded_multi_stmt_txn_jscore_passthrough
distros:
- rhel80-medium
- name: .sharding .jscore !.wo_snapshot !.multi_stmt
- name: .sharding .common !.multiversion !.csrs
- name: snmp
- name: .ssl
- name: .updatefuzzer
- name: enterprise-windows-inmem
display_name: Enterprise Windows (inMemory)
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- windows-vsCurrent-small
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug msi
exe: ".exe"
content_type: application/zip
compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10 --use-diagnostic-latches=on
num_scons_link_jobs_available: 0.25
python: '/cygdrive/c/python/python37/python.exe'
test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,incompatible_with_windows_tls
ext: zip
scons_cache_scope: shared
multiversion_platform: windows
multiversion_edition: enterprise
large_distro_name: windows-vsCurrent-large
tasks:
- name: compile_test_and_package_serial_TG
distros:
- windows-vsCurrent-large
- name: compile_test_and_package_parallel_unittest_stream_TG
distros:
- windows-vsCurrent-large
- name: .aggfuzzer .common
- name: audit
- name: auth_audit_gen
- name: causally_consistent_jscore_txns_passthrough
- name: .concurrency .common
distros:
- windows-vsCurrent-large
- name: concurrency_replication_causal_consistency_gen
- name: initial_sync_fuzzer_gen
- name: .jscore .common !.decimal !.sharding
- name: .jstestfuzz .common !.flow_control # Flow control jstestfuzz take longer.
- name: .read_write_concern .linearize !.durable_history
- name: replica_sets_auth_gen
- name: replica_sets_jscore_passthrough_gen
- name: replica_sets_multi_stmt_txn_jscore_passthrough
- name: sasl
- name: .sharding .txns
- name: sharding_auth_gen
- name: sharding_auth_audit_gen
- name: snmp
- name: .ssl
- name: .resharding_fuzzer

File diff suppressed because it is too large Load Diff

View File

@ -1,85 +0,0 @@
buildvariants:
- name: enterprise-windows-ninja
display_name: "Ninja Build: Enterprise Windows"
cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
expansions:
compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5") --win-version-min=win10
tasks:
- name: compile_ninja_next_TG
distros:
- windows-vsCurrent-large
- name: compile_ninja_TG
distros:
- windows-vsCurrent-large
- name: macos-enterprise-ninja
display_name: "Ninja Build: macOS Enterprise"
cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
run_on:
- macos-11
expansions:
compile_env: DEVELOPER_DIR=/Applications/Xcode13.app
compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars
tasks:
- name: compile_ninja_next_TG
- name: compile_ninja_TG
- name: ubuntu1804-ninja-build-profiles
display_name: "Ninja Build Profiles: Ubuntu 18.04"
cron: "0 4 * * 0" # # Run once a week to ensure no failures introduced to build profiles
run_on:
- ubuntu1804-small
stepback: false
expansions:
compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --modules=
test_flags: --enableEnterpriseTests=off
repo_edition: enterprise
has_packages: false
tasks:
- name: compile_ninja_default_profile_TG
distros:
- ubuntu1804-xlarge
- name: compile_ninja_opt_profile_TG
distros:
- ubuntu1804-xlarge
- name: compile_ninja_san_profile_TG
distros:
- ubuntu1804-xlarge
- name: compile_ninja_fast_profile_TG
distros:
- ubuntu1804-xlarge
- name: enterprise-rhel-8-64-bit-dynamic-required-ninja
display_name: "Ninja Build: Enterprise RHEL 8"
cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
run_on:
- rhel80-small
expansions:
compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
has_packages: false
tasks:
- name: compile_ninja_next_TG
distros:
- rhel80-xlarge
- name: compile_ninja_TG
distros:
- rhel80-xlarge
- name: enterprise-rhel8-ninja
display_name: "Ninja Build: RHEL 8"
cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds
run_on:
- rhel80-small
stepback: false
expansions:
compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
repo_edition: enterprise
has_packages: false
tasks:
- name: compile_ninja_next_TG
distros:
- rhel80-build
- name: compile_ninja_TG
distros:
- rhel80-build

View File

@ -1,168 +0,0 @@
# Build variant definitions for vanilla sanitizers that can be used across
# release and dev environments.
buildvariants:
- name: rhel8-asan
display_name: ~ ASAN RHEL 8
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel80-build
stepback: true
expansions:
lang_environment: LANG=C
san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on
multiversion_platform: rhel80
multiversion_platform_50_or_later: rhel8
multiversion_edition: enterprise
resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build.
hang_analyzer_dump_core: false
scons_cache_scope: shared
test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling
separate_debug: off
tasks:
- name: compile_test_and_package_serial_TG
- name: compile_test_and_package_parallel_unittest_stream_TG
- name: .aggfuzzer .common
- name: .jstestfuzz !.initsync
- name: rhel8-debug-asan
display_name: ~ ASAN Enterprise RHEL 8 DEBUG
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel80-build
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
lang_environment: LANG=C
# If you add anything to san_options, make sure the appropriate changes are
# also made to SConstruct.
san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo)
test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling,requires_increased_memlock_limits,requires_latch_analyzer
multiversion_platform: rhel80
multiversion_platform_50_or_later: rhel8
multiversion_edition: enterprise
resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build.
hang_analyzer_dump_core: false
scons_cache_scope: shared
exec_timeout_secs: 14400 # 3 hour timeout
separate_debug: off
large_distro_name: rhel80-build
tasks:
- name: compile_test_and_package_serial_TG
- name: compile_test_and_package_parallel_unittest_stream_TG
- name: compile_benchmarks
- name: compile_integration_and_test_parallel_stream_TG
- name: .aggregation
- name: .auth
- name: audit
- name: .benchmarks
- name: .causally_consistent !.wo_snapshot
- name: .change_streams
- name: .misc_js
- name: .concurrency !.ubsan !.no_txns !.kill_terminate
- name: .encrypt
- name: external_auth
- name: external_auth_aws
- name: initial_sync_fuzzer_gen
- name: .jscore .common
- name: jsCore_minimum_batch_size
- name: jsCore_txns_large_txns_format
- name: json_schema
- name: .logical_session_cache
- name: .multi_shard .common
- name: multiversion_gen
- name: .multiversion_fuzzer
- name: .multiversion_passthrough
- name: .query_fuzzer
- name: .random_multiversion_ds
- name: .read_write_concern
- name: replica_sets_large_txns_format_gen
- name: replica_sets_large_txns_format_jscore_passthrough
- name: .replica_sets !.multi_oplog
- name: .resharding_fuzzer
- name: .retry
- name: .read_only
- name: .rollbackfuzzer
- name: .updatefuzzer
- name: sasl
- name: secondary_reads_passthrough_gen
- name: session_jscore_passthrough
- name: .sharding .jscore !.wo_snapshot
- name: .sharding .common !.csrs
- name: snmp
- name: .watchdog
- name: .serverless
- name: unittest_shell_hang_analyzer_gen
- name: .updatefuzzer
- name: server_discovery_and_monitoring_json_test_TG
- name: server_selection_json_test_TG
- name: rhel8-debug-ubsan
display_name: ~ UBSAN Enterprise RHEL 8 DEBUG
cron: "0 4 * * *" # From the ${project_nightly_cron} parameter.
run_on:
- rhel80-build
stepback: false
expansions:
additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
lang_environment: LANG=C
# If you add anything to san_options, make sure the appropriate changes are
# also made to SConstruct.
san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer"
compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on
test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits
multiversion_platform: rhel80
multiversion_platform_50_or_later: rhel8
multiversion_edition: enterprise
resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build.
scons_cache_scope: shared
separate_debug: off
large_distro_name: rhel80-build
tasks:
- name: compile_test_and_package_serial_TG
- name: compile_test_and_package_parallel_unittest_stream_TG
- name: compile_benchmarks
- name: compile_integration_and_test_parallel_stream_TG
- name: .aggregation
- name: .auth
- name: audit
- name: .benchmarks
- name: .causally_consistent !.wo_snapshot
- name: .change_streams
- name: .misc_js
- name: .concurrency !.no_txns !.kill_terminate
- name: disk_wiredtiger
- name: .encrypt
- name: initial_sync_fuzzer_gen
- name: .jscore .common
- name: jsCore_minimum_batch_size
- name: jsCore_txns_large_txns_format
- name: json_schema
- name: .logical_session_cache .one_sec
- name: .multi_shard .common
- name: multiversion_gen
- name: .multiversion_fuzzer
- name: .multiversion_passthrough
- name: .random_multiversion_ds
- name: .read_write_concern
- name: replica_sets_large_txns_format_gen
- name: replica_sets_large_txns_format_jscore_passthrough
- name: .replica_sets !.multi_oplog
- name: .resharding_fuzzer
- name: .retry
- name: .rollbackfuzzer
- name: .read_only
- name: sasl
- name: secondary_reads_passthrough_gen
- name: session_jscore_passthrough
- name: .sharding .jscore !.wo_snapshot
- name: .sharding .common !.csrs
- name: snmp
- name: .updatefuzzer
- name: .serverless
- name: watchdog_wiredtiger
- name: server_discovery_and_monitoring_json_test_TG
- name: server_selection_json_test_TG

View File

@ -1,21 +0,0 @@
# Build variant to generate tasks for evergreen versions.
#
# Updates to this file may also need to appear in etc/system_perf_yml_components/variants/task_generation.yml,
# which is the same but excludes resmoke task generation tasks.
#
buildvariants:
- name: generate-tasks-for-version
display_name: "Generate tasks for evergreen version"
batchtime: 0
activate: true
run_on:
- rhel80-medium
tasks:
- name: version_gen
distros:
- ubuntu2004-medium
- name: version_burn_in_gen
distros:
- ubuntu2004-medium
- name: version_expansions_gen

View File

@ -180,7 +180,7 @@ the storage mechanism is listed as a sub-bullet below.
authentication. GSSAPI is the communication method used to communicate with Kerberos servers and
with clients. When initializing this auth mechanism, the server tries to acquire its credential
information from the KDC by calling
[`tryAcquireServerCredential`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/sasl/mongo_gssapi.h#L36).
[`tryAcquireServerCredential`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/sasl/mongo_gssapi.h#L36).
If this is not approved, the server fasserts and the mechanism is not registered. On Windows,
SChannel provides a `GSSAPI` library for the server to use. On other platforms, the Cyrus SASL
library is used to make calls to the KDC (Kerberos key distribution center).
@ -607,30 +607,30 @@ The user must supply roles when running the `createUser` command. Roles are stor
LDAP authorization is an external method of getting roles. When a user authenticates using LDAP,
there are roles stored in the User document specified by the LDAP system. The LDAP system relies on
the
[`AuthzManagerExternalStateLDAP`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.h)
[`AuthzManagerExternalStateLDAP`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.h)
to make external requests to the LDAP server. The `AuthzManagerExternalStateLDAP` wraps the
`AuthzManagerExternalStateLocal` for the current process, initially attempting to route all
Authorization requests to LDAP and falling back on Local Authorization. LDAP queries are generated
from
[`UserRequest`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.cpp#L75-L113)
[`UserRequest`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.cpp#L75-L113)
objects, passing just the username into the query. If a user has specified the `userToDNMapping`
server parameter, the `AuthorizationManager` calls the LDAPManager to transform the usernames into
names that the LDAP server can understand. The LDAP subsystem relies on a complicated string
escaping sequence, which is handled by the LDAPQuery class. After LDAP has returned the `User`
document, it resolves role names into privileges by dispatching a call to
[`Local::getUserObject`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.7.0/src/ldap/authz_manager_external_state_ldap.cpp#L110-L123)
[`Local::getUserObject`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.7.0/src/ldap/authz_manager_external_state_ldap.cpp#L110-L123)
with a `UserRequest` struct containing a set of roles to be resolved.
Connections to LDAP servers are made by the `LDAPManager` through the
[`LDAPRunner`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/ldap_runner.h)
[`LDAPRunner`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/ldap_runner.h)
by calling `bindAsUser()`. `BindAsUser()` attempts to set up a connection to the LDAP server using
connection parameters specified through the command line when starting the process.The
[`LDAPConnectionFactory`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/connections/ldap_connection_factory.h)
[`LDAPConnectionFactory`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/connections/ldap_connection_factory.h)
is the class that is actually tasked with establishing a connection and sending raw bytes over the
wire to the LDAP server, all other classes decompose the information to send and use the factory to
actually send the information. The `LDAPConnectionFactory` has its own thread pool and executor to
drive throughput for authorization. LDAP has an
[`LDAPUserCacheInvalidator`](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/ldap_user_cache_invalidator_job.h)
[`LDAPUserCacheInvalidator`](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/ldap_user_cache_invalidator_job.h)
that periodically sweeps the `AuthorizationManager` and deletes user entries that have `$external` as
their authentication database.
@ -638,7 +638,7 @@ There are a few thread safety concerns when making connections to the LDAP serve
LibLDAP to make connections to the LDAP server. LibLDAP comes without any thread safety guarantees,
so all the calls to libLDAP are wrapped with mutexes to ensure thread safety when connecting to LDAP
servers on certain distros. The logic to see whether libLDAP is thread-safe lives
[here](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/connections/openldap_connection.cpp#L348-L378).
[here](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/connections/openldap_connection.cpp#L348-L378).
#### <a name="x509azn"></a>X.509 Authorization
@ -717,7 +717,7 @@ Refer to the following links for definitions of the Classes referenced in this d
| `AuthorizationManager` | [mongo/db/auth/authorization\_manager.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/auth/authorization_manager.h) | Interface to external state providers |
| `AuthorizationSession` | [mongo/db/auth/authorization\_session.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/auth/authorization_session.h) | Representation of currently authenticated and authorized users on the `Client` connection |
| `AuthzManagerExternalStateLocal` | [.../authz\_manager\_external\_state\_local.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/auth/authz_manager_external_state_local.h) | `Local` implementation of user/role provider |
| `AuthzManagerExternalStateLDAP` | [.../authz\_manager\_external\_state\_ldap.h](https://github.com/10gen/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.h) | `LDAP` implementation of users/role provider |
| `AuthzManagerExternalStateLDAP` | [.../authz\_manager\_external\_state\_ldap.h](https://github.com/mongodb/mongo-enterprise-modules/blob/r4.4.0/src/ldap/authz_manager_external_state_ldap.h) | `LDAP` implementation of users/role provider |
| `Client` | [mongo/db/client.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/client.h) | An active client session, typically representing a remote driver or shell |
| `Privilege` | [mongo/db/auth/privilege.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/auth/privilege.h) | A set of `ActionType`s permitted on a particular `resource' |
| `ResourcePattern` | [mongo/db/auth/resource\_pattern.h](https://github.com/mongodb/mongo/blob/r4.4.0/src/mongo/db/auth/resource_pattern.h) | A reference to a namespace, db, collection, or cluster to apply a set of `ActionType` privileges to |

View File

@ -949,7 +949,7 @@ For retry images saved in the image collection, the source will "downconvert" op
`needsRetryImage: true` into two oplog entries, simulating the old format. As chunk migrations use
internal commands, [this downconverting procedure](https://github.com/mongodb/mongo/blob/0beb0cacfcaf7b24259207862e1d0d489e1c16f1/src/mongo/db/s/session_catalog_migration_source.cpp#L58-L97)
is installed under the hood. For resharding and tenant migrations, a new aggregation stage,
[_internalFindAndModifyImageLookup](https://github.com/10gen/mongo/blob/e27dfa10b994f6deff7c59a122b87771cdfa8aba/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp#L61),
[_internalFindAndModifyImageLookup](https://github.com/mongodb/mongo/blob/e27dfa10b994f6deff7c59a122b87771cdfa8aba/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp#L61),
was introduced to perform the identical substitution. In order for this stage to have a valid timestamp
to assign to the forged no-op oplog entry as result of the "downconvert", we must always assign an
extra oplog slot when writing the original retryable findAndModify oplog entry with

View File

@ -250,7 +250,11 @@ typename FlatBSONStore<Element, Value>::Iterator FlatBSONStore<Element, Value>::
auto it = begin();
auto itEnd = end();
for (; it != itEnd; ++it) {
(*_pos->_fieldNameToIndex)[it->fieldName().toString()] = it._pos->_offsetParent;
uassert(
12602100,
"Duplicate field names cannot be present in the same FlatBSON object",
_pos->_fieldNameToIndex->try_emplace(it->fieldName().toString(), it._pos->_offsetParent)
.second);
}
// Retry the search now when the map is created.
@ -280,7 +284,12 @@ FlatBSONStore<Element, Value>::Obj::insert(FlatBSONStore<Element, Value>::Iterat
// Also store our offset in the fast lookup map if it is available.
if (_pos->_fieldNameToIndex) {
_pos->_fieldNameToIndex->emplace(inserted->_element.fieldName(), inserted->_offsetParent);
uassert(
12602101,
"Duplicate field names cannot be present in the same FlatBSON object",
_pos->_fieldNameToIndex
->try_emplace(inserted->_element.fieldName().toString(), inserted->_offsetParent)
.second);
}
// We need to traverse the hiearchy up to the root and modify stored offsets to account for

View File

@ -261,6 +261,37 @@ TEST(MinMax, SearchLookupMap) {
ASSERT_EQ(obj.search(obj.begin(), "50")->fieldName(), "50");
}
TEST(MinMax, DuplicateFieldNamesWithLookupMap) {
MinMaxStore minmax;
auto obj = minmax.root();
// Insert 12 (kMaxLinearSearchLength) distinct fields ("0".."11") followed by two duplicate "a"
// entries. This will trigger the lookup map internally in flat_bson.
for (int i = 0; i < 12; ++i) {
obj.insert(obj.end(), std::to_string(i));
}
obj.insert(obj.end(), "a");
obj.insert(obj.end(), "a");
// Try to search for "a", this will trigger the lookup map internally in flat_bson as we fail to
// find it within 'kMaxLinearSearchLength' attempts. The map cannot contain duplicates so this
// search is well defined and throws.
ASSERT_THROWS(obj.search(obj.begin(), "a"), AssertionException);
// Try to insert another duplicate which will throw earlier as the lookup map exists and needs
// to be maintained.
obj.insert(obj.begin(), "x");
ASSERT_THROWS(obj.insert(obj.begin(), "x"), AssertionException);
// Searching for "a" or "x" is possible as we inserted one of them into the map.
auto found = obj.search(obj.begin(), "a");
ASSERT(found != obj.end());
ASSERT_EQ(found->fieldName(), "a");
found = obj.search(obj.begin(), "x");
ASSERT(found != obj.end());
ASSERT_EQ(found->fieldName(), "x");
}
} // namespace
} // namespace mongo::timeseries