From d5122c7b20aa339232497bd9658b00899c7b0572 Mon Sep 17 00:00:00 2001 From: Colin Stolley <3535347+ccstolley@users.noreply.github.com> Date: Fri, 6 Feb 2026 13:09:25 -0600 Subject: [PATCH] SERVER-116272: Introduce 'validated' document schema validation level (#46706) Co-authored-by: Niels Lohmann GitOrigin-RevId: 367679c83d10a0ae7c312d545da5130573694d82 --- .../doc_validation/validated_limitations.js | 194 ++++++++++++++++++ .../schema_validation_level.js | 35 ++++ src/mongo/db/collection_compact.cpp | 2 +- src/mongo/db/collection_crud/capped_utils.cpp | 2 +- .../collection_crud/collection_write_path.cpp | 4 +- src/mongo/db/commands/apply_ops_cmd.cpp | 2 +- .../db/commands/oplog_application_checks.cpp | 2 +- .../db/commands/query_cmd/bulk_write.cpp | 4 +- .../db/commands/query_cmd/find_and_modify.cpp | 4 +- ..._feature_compatibility_version_command.cpp | 49 +++-- .../db/query/write_ops/write_ops_exec.cpp | 12 +- src/mongo/db/repair.cpp | 2 +- src/mongo/db/repl/oplog_applier_impl_test.cpp | 11 +- .../repl/oplog_applier_impl_test_fixture.cpp | 5 +- src/mongo/db/repl/oplog_applier_utils.cpp | 4 +- src/mongo/db/repl/oplog_buffer_collection.cpp | 2 +- src/mongo/db/repl/storage_interface_impl.cpp | 2 +- .../db/repl/storage_interface_impl_test.cpp | 4 +- src/mongo/db/repl/storage_timestamp_test.cpp | 4 +- .../db/repl/transaction_oplog_application.cpp | 2 +- src/mongo/db/s/migration_batch_inserter.cpp | 2 +- .../s/migration_chunk_cloner_source_test.cpp | 13 +- .../db/s/migration_destination_manager.cpp | 2 +- src/mongo/db/server_feature_flags.idl | 5 + .../ddl/clone_catalog_data_command.cpp | 2 +- src/mongo/db/shard_role/ddl/collmod_cmd.cpp | 25 ++- src/mongo/db/shard_role/ddl/create.idl | 2 +- .../db/shard_role/ddl/create_command.cpp | 25 ++- .../db/shard_role/shard_catalog/BUILD.bazel | 1 + .../db/shard_role/shard_catalog/coll_mod.cpp | 16 +- .../db/shard_role/shard_catalog/collection.h | 35 +++- .../shard_catalog/collection_impl.cpp | 152 +++++--------- .../shard_catalog/collection_impl.h | 10 +- .../shard_catalog/collection_mock.h | 12 +- .../shard_catalog/collection_options.idl | 1 + .../shard_catalog/document_validation.h | 54 +++-- .../document_validation_helpers.cpp | 92 +++++++++ .../document_validation_helpers.h | 44 ++++ .../shard_catalog/rename_collection.cpp | 6 +- .../shard_catalog/virtual_collection_impl.h | 13 +- .../db/timeseries/timeseries_write_util.cpp | 2 +- .../timeseries_write_ops_internal.cpp | 2 +- 42 files changed, 624 insertions(+), 238 deletions(-) create mode 100644 jstests/core/query/doc_validation/validated_limitations.js create mode 100644 jstests/multiVersion/genericBinVersion/query-execution/schema_validation_level.js create mode 100644 src/mongo/db/shard_role/shard_catalog/document_validation_helpers.cpp create mode 100644 src/mongo/db/shard_role/shard_catalog/document_validation_helpers.h diff --git a/jstests/core/query/doc_validation/validated_limitations.js b/jstests/core/query/doc_validation/validated_limitations.js new file mode 100644 index 00000000000..0c19eb7a795 --- /dev/null +++ b/jstests/core/query/doc_validation/validated_limitations.js @@ -0,0 +1,194 @@ +/** + * Test that constraints on collections with validation level 'validated' are correctly enforced. + * + * @tags: [ + * # 'validated' level was introduced in 8.3. + * requires_fcv_83, + * featureFlagValidatedValidationLevel, + * ] + */ + +const dbName = "validated_tests"; +const collName = "test"; +const validator = {a: {$exists: true}}; +const myDb = db.getSiblingDB(dbName); + +function createValidatedCollection() { + myDb.runCommand({drop: collName, writeConcern: {w: "majority"}}); + assert.commandWorked( + myDb.createCollection(collName, { + validator: validator, + validationLevel: "validated", + validationAction: "error", + writeConcern: {w: "majority"}, + }), + ); +} + +function validationActionMustNotBeWarn() { + myDb.runCommand({drop: collName, writeConcern: {w: "majority"}}); + assert.commandFailed( + myDb.createCollection(collName, { + validator: validator, + validationLevel: "validated", + validationAction: "warn", + writeConcern: {w: "majority"}, + }), + ); + + createValidatedCollection(); + assert.commandFailed(myDb.runCommand({"collMod": collName, validationAction: "warn"})); + assert.commandWorked(myDb.runCommand({"collMod": collName, validationAction: "errorAndLog"})); + + // Warn should be ok for any other level. + assert.commandWorked( + myDb.runCommand({ + "collMod": collName, + validationLevel: "strict", + validator: {b: {$exists: true}}, + writeConcern: {w: "majority"}, + validationAction: "error", + }), + ); + assert.commandWorked(myDb.runCommand({"collMod": collName, validationAction: "warn"})); + + assert.commandWorked( + myDb.runCommand({ + "collMod": collName, + validationLevel: "moderate", + validator: {b: {$exists: true}}, + writeConcern: {w: "majority"}, + validationAction: "error", + }), + ); + assert.commandWorked(myDb.runCommand({"collMod": collName, validationAction: "warn"})); + + assert.commandWorked( + myDb.runCommand({ + "collMod": collName, + validationLevel: "off", + validator: {b: {$exists: true}}, + writeConcern: {w: "majority"}, + validationAction: "error", + }), + ); + assert.commandWorked(myDb.runCommand({"collMod": collName, validationAction: "warn"})); +} + +function noValidatedOnExisting() { + myDb.runCommand({drop: collName, writeConcern: {w: "majority"}}); + assert.commandWorked(myDb.createCollection(collName, {writeConcern: {w: "majority"}})); + assert.commandFailed( + myDb.runCommand({ + "collMod": collName, + validator: validator, + validationLevel: "validated", + validationAction: "error", + writeConcern: {w: "majority"}, + }), + ); + assert.commandFailed( + myDb.runCommand({ + "collMod": collName, + validationLevel: "validated", + validationAction: "warn", + writeConcern: {w: "majority"}, + }), + ); + assert.commandFailed( + myDb.runCommand({ + "collMod": collName, + validationLevel: "validated", + writeConcern: {w: "majority"}, + }), + ); +} + +function noSchemaRuleChanges() { + createValidatedCollection(); + assert.commandFailed( + myDb.runCommand({"collMod": collName, writeConcern: {w: "majority"}, validator: {b: {$exists: true}}}), + ); + // Rules changes are ok if we also downgrade to strict. + assert.commandWorked( + myDb.runCommand({ + "collMod": collName, + validationLevel: "strict", + writeConcern: {w: "majority"}, + validator: {b: {$exists: true}}, + }), + ); +} + +function downgradeValidatedToStrict() { + createValidatedCollection(); + assert.commandWorked( + myDb.runCommand({ + "collMod": collName, + validationLevel: "strict", + writeConcern: {w: "majority"}, + }), + ); +} + +function noNonConformingDocs() { + createValidatedCollection(); + assert.commandFailed( + myDb.runCommand({ + insert: collName, + documents: [{b: 1}], + writeConcern: {w: "majority"}, + }), + ); +} + +function noBypassDocValidation() { + createValidatedCollection(); + // conforming doc (bypassDocumentValidation:true always fails on validated collections) + assert.commandFailed( + myDb.runCommand({ + insert: collName, + documents: [{a: 1}], + bypassDocumentValidation: true, + writeConcern: {w: "majority"}, + }), + ); + // non-conforming doc + assert.commandFailed( + myDb.runCommand({ + insert: collName, + documents: [{c: 1}], + bypassDocumentValidation: true, + writeConcern: {w: "majority"}, + }), + ); + + // conforming doc + assert.commandWorked( + myDb.runCommand({ + insert: collName, + documents: [{a: 1}], + bypassDocumentValidation: false, + writeConcern: {w: "majority"}, + }), + ); + // non-conforming doc + assert.commandFailed( + myDb.runCommand({ + insert: collName, + documents: [{c: 1}], + bypassDocumentValidation: false, + writeConcern: {w: "majority"}, + }), + ); +} + +validationActionMustNotBeWarn(); +downgradeValidatedToStrict(); +noSchemaRuleChanges(); +noValidatedOnExisting(); +noNonConformingDocs(); +noBypassDocValidation(); + +// avoid downgrade problems (validated collections can't be downgraded) +myDb.runCommand({drop: collName, writeConcern: {w: "majority"}}); diff --git a/jstests/multiVersion/genericBinVersion/query-execution/schema_validation_level.js b/jstests/multiVersion/genericBinVersion/query-execution/schema_validation_level.js new file mode 100644 index 00000000000..146b0a3b16c --- /dev/null +++ b/jstests/multiVersion/genericBinVersion/query-execution/schema_validation_level.js @@ -0,0 +1,35 @@ +/** + * Test edge cases for the schema validation level 'validated' + */ +import "jstests/multiVersion/libs/multi_rs.js"; + +import {ReplSetTest} from "jstests/libs/replsettest.js"; + +const rst = new ReplSetTest({ + nodes: 2, + nodeOptions: {binVersion: "latest", setParameter: {featureFlagValidatedValidationLevel: true}}, +}); +rst.startSet(); +rst.initiate(); + +const db = rst.getPrimary().getDB("test"); + +assert.commandWorked( + db.createCollection("test", {validator: {a: 1}, validationAction: "error", validationLevel: "validated"}), +); + +// Can't downgrade FCV to lastLTS when collection has validated validation level. +assert.commandFailedWithCode( + rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true}), + ErrorCodes.CannotDowngrade, +); + +assert.commandWorked(db.runCommand({collMod: "test", validationLevel: "strict"})); + +// After removing validated validation level, we should eventually be able to downgrade +assert.soon(() => { + assert.commandWorked(rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true})); + return true; +}); + +rst.stopSet(); diff --git a/src/mongo/db/collection_compact.cpp b/src/mongo/db/collection_compact.cpp index 67918d65ee2..843f325ea9b 100644 --- a/src/mongo/db/collection_compact.cpp +++ b/src/mongo/db/collection_compact.cpp @@ -69,7 +69,7 @@ using logv2::LogComponent; StatusWith compactCollection(OperationContext* opCtx, const CompactOptions& options, const CollectionPtr& collection) { - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); auto collectionNss = collection->ns(); auto recordStore = collection->getRecordStore(); diff --git a/src/mongo/db/collection_crud/capped_utils.cpp b/src/mongo/db/collection_crud/capped_utils.cpp index f0ac92597b0..50df7de64f9 100644 --- a/src/mongo/db/collection_crud/capped_utils.cpp +++ b/src/mongo/db/collection_crud/capped_utils.cpp @@ -151,7 +151,7 @@ void cloneCollectionAsCapped(OperationContext* opCtx, BSONObj objToClone; RecordId loc; - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); auto replCoord = repl::ReplicationCoordinator::get(opCtx); bool isOplogDisabledForCappedCollection = replCoord->isOplogDisabledFor(opCtx, toNss); diff --git a/src/mongo/db/collection_crud/collection_write_path.cpp b/src/mongo/db/collection_crud/collection_write_path.cpp index 164f29170ff..4c6ba57289f 100644 --- a/src/mongo/db/collection_crud/collection_write_path.cpp +++ b/src/mongo/db/collection_crud/collection_write_path.cpp @@ -648,8 +648,8 @@ void updateDocument(OperationContext* opCtx, { auto status = collection->checkValidationAndParseResult(opCtx, newDoc); if (!status.isOK()) { - if (validationLevelOrDefault(collection->getCollectionOptions().validationLevel) == - ValidationLevelEnum::strict) { + if (validationLevelIsMandatory( + validationLevelOrDefault(collection->getCollectionOptions().validationLevel))) { uassertStatusOK(status); } // moderate means we have to check the old doc diff --git a/src/mongo/db/commands/apply_ops_cmd.cpp b/src/mongo/db/commands/apply_ops_cmd.cpp index b6a13a64f44..35276186233 100644 --- a/src/mongo/db/commands/apply_ops_cmd.cpp +++ b/src/mongo/db/commands/apply_ops_cmd.cpp @@ -252,7 +252,7 @@ public: validateApplyOpsCommand(cmdObj); - boost::optional maybeDisableValidation; + boost::optional maybeDisableValidation; if (shouldBypassDocumentValidationForCommand(cmdObj)) maybeDisableValidation.emplace(opCtx); diff --git a/src/mongo/db/commands/oplog_application_checks.cpp b/src/mongo/db/commands/oplog_application_checks.cpp index 29778677846..601c3055472 100644 --- a/src/mongo/db/commands/oplog_application_checks.cpp +++ b/src/mongo/db/commands/oplog_application_checks.cpp @@ -290,7 +290,7 @@ Status OplogApplicationChecks::checkAuthForOperation(OperationContext* opCtx, } fassert(40314, validity == OplogApplicationValidity::kOk); - boost::optional maybeDisableValidation; + boost::optional maybeDisableValidation; if (shouldBypassDocumentValidationForCommand(cmdObj)) maybeDisableValidation.emplace(opCtx); diff --git a/src/mongo/db/commands/query_cmd/bulk_write.cpp b/src/mongo/db/commands/query_cmd/bulk_write.cpp index 346b89e07d6..a525054dd80 100644 --- a/src/mongo/db/commands/query_cmd/bulk_write.cpp +++ b/src/mongo/db/commands/query_cmd/bulk_write.cpp @@ -1875,8 +1875,8 @@ BulkWriteReply performWrites(OperationContext* opCtx, const BulkWriteCommandRequ } const auto& bypassDocumentValidation = req.getBypassDocumentValidation(); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - bypassDocumentValidation); + DisableDocumentSchemaValidationRequestedByUserIfTrue docSchemaValidationDisabler( + opCtx, bypassDocumentValidation); const auto& firstNsInfo = req.getNsInfo()[0]; const bool fleCrudProcessed = write_ops_exec::getFleCrudProcessed( diff --git a/src/mongo/db/commands/query_cmd/find_and_modify.cpp b/src/mongo/db/commands/query_cmd/find_and_modify.cpp index c10a7e0a895..7b067d5d7b5 100644 --- a/src/mongo/db/commands/query_cmd/find_and_modify.cpp +++ b/src/mongo/db/commands/query_cmd/find_and_modify.cpp @@ -506,8 +506,8 @@ write_ops::FindAndModifyCommandReply CmdFindAndModify::Invocation::typedRun( auto fleCrudProcessed = write_ops_exec::getFleCrudProcessed( opCtx, req.getEncryptionInformation(), nsString.tenantId()); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); + DisableDocumentSchemaValidationRequestedByUserIfTrue docSchemaValidationDisabler( + opCtx, disableDocumentValidation); DisableSafeContentValidationIfTrue safeContentValidationDisabler( opCtx, disableDocumentValidation, fleCrudProcessed); diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index fa93072731c..a14655715e9 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -1465,28 +1465,39 @@ private: "Simulated dry-run validation failure via fail point."); } - if (gFeatureFlagErrorAndLogValidationAction.isDisabledOnTargetFCVButEnabledOnOriginalFCV( - requestedVersion, originalVersion)) { + bool errorAndLogValidationDisabled = + (gFeatureFlagErrorAndLogValidationAction.isDisabledOnTargetFCVButEnabledOnOriginalFCV( + requestedVersion, originalVersion)); + bool validatedValidationLevelDisabled = + (gFeatureFlagValidatedValidationLevel.isDisabledOnTargetFCVButEnabledOnOriginalFCV( + requestedVersion, originalVersion)); + if (errorAndLogValidationDisabled || validatedValidationLevelDisabled) { for (const auto& dbName : DatabaseHolder::get(opCtx)->getNames()) { Lock::DBLock dbLock(opCtx, dbName, MODE_IS); catalog::forEachCollectionFromDb( - opCtx, - dbName, - MODE_IS, - [&](const Collection* collection) -> bool { - uasserted( - ErrorCodes::CannotDowngrade, - fmt::format( - "Cannot downgrade the cluster when there are collections with " - "'errorAndLog' validation action. Please unset the option or " - "drop the collection(s) before downgrading. First detected " - "collection with 'errorAndLog' enabled: {} (UUID: {}).", - collection->ns().toStringForErrorMsg(), - collection->uuid().toString())); - }, - [&](const Collection* collection) { - return collection->getValidationAction() == - ValidationActionEnum::errorAndLog; + opCtx, dbName, MODE_IS, [&](const Collection* collection) -> bool { + uassert(ErrorCodes::CannotDowngrade, + fmt::format( + "Cannot downgrade the cluster when there are collections with " + "'errorAndLog' validation action. Please unset the option or " + "drop the collection(s) before downgrading. First detected " + "collection with 'errorAndLog' enabled: {} (UUID: {}).", + collection->ns().toStringForErrorMsg(), + collection->uuid().toString()), + collection->getValidationAction() != + ValidationActionEnum::errorAndLog); + + uassert(ErrorCodes::CannotDowngrade, + fmt::format( + "Cannot downgrade the cluster when there are collections with " + "'validated' validation level. Please unset the option or " + "drop the collection(s) before downgrading. First detected " + "collection with 'validated' enabled: {} (UUID: {}).", + collection->ns().toStringForErrorMsg(), + collection->uuid().toString()), + collection->getValidationLevel() != ValidationLevelEnum::validated); + + return true; }); } } diff --git a/src/mongo/db/query/write_ops/write_ops_exec.cpp b/src/mongo/db/query/write_ops/write_ops_exec.cpp index 36ea4eae1e0..ed591599e7b 100644 --- a/src/mongo/db/query/write_ops/write_ops_exec.cpp +++ b/src/mongo/db/query/write_ops/write_ops_exec.cpp @@ -1217,8 +1217,8 @@ WriteResult performInserts( const auto [disableDocumentValidation, fleCrudProcessed] = getDocumentValidationFlags( opCtx, wholeOp.getWriteCommandRequestBase(), wholeOp.getDbName().tenantId()); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); + DisableDocumentSchemaValidationRequestedByUserIfTrue docSchemaValidationDisabler( + opCtx, disableDocumentValidation); DisableSafeContentValidationIfTrue safeContentValidationDisabler( opCtx, disableDocumentValidation, fleCrudProcessed); @@ -1821,8 +1821,8 @@ WriteResult performUpdates( const auto [disableDocumentValidation, fleCrudProcessed] = getDocumentValidationFlags( opCtx, wholeOp.getWriteCommandRequestBase(), wholeOp.getDbName().tenantId()); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); + DisableDocumentSchemaValidationRequestedByUserIfTrue docSchemaValidationDisabler( + opCtx, disableDocumentValidation); DisableSafeContentValidationIfTrue safeContentValidationDisabler( opCtx, disableDocumentValidation, fleCrudProcessed); @@ -2157,8 +2157,8 @@ WriteResult performDeletes( const auto [disableDocumentValidation, fleCrudProcessed] = getDocumentValidationFlags( opCtx, wholeOp.getWriteCommandRequestBase(), wholeOp.getDbName().tenantId()); - DisableDocumentSchemaValidationIfTrue docSchemaValidationDisabler(opCtx, - disableDocumentValidation); + DisableDocumentSchemaValidationRequestedByUserIfTrue docSchemaValidationDisabler( + opCtx, disableDocumentValidation); DisableSafeContentValidationIfTrue safeContentValidationDisabler( opCtx, disableDocumentValidation, fleCrudProcessed); diff --git a/src/mongo/db/repair.cpp b/src/mongo/db/repair.cpp index 4d09e5a9ed0..841f59183dc 100644 --- a/src/mongo/db/repair.cpp +++ b/src/mongo/db/repair.cpp @@ -176,7 +176,7 @@ Status repairCollections(OperationContext* opCtx, namespace repair { Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const DatabaseName& dbName) { - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); // We must hold some form of lock here invariant(shard_role_details::getLocker(opCtx)->isW()); diff --git a/src/mongo/db/repl/oplog_applier_impl_test.cpp b/src/mongo/db/repl/oplog_applier_impl_test.cpp index 79629cb72ab..c48c21e0223 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test.cpp @@ -887,7 +887,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryToInvalidateChangeStreamPreImages) { // Apply the oplog entry. { repl::UnreplicatedWritesBlock uwb(_opCtx.get()); - DisableDocumentValidation validationDisabler(_opCtx.get()); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx.get()); ASSERT_THROWS(applyOplogEntryOrGroupedInserts(_opCtx.get(), ApplierOperation{&invalidateOp}, OplogApplication::Mode::kInitialSync, @@ -964,7 +964,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryToInvalidateNonModPreImages) { // Apply the oplog entry. { repl::UnreplicatedWritesBlock uwb(_opCtx.get()); - DisableDocumentValidation validationDisabler(_opCtx.get()); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx.get()); ASSERT_NOT_OK(applyOplogEntryOrGroupedInserts(_opCtx.get(), ApplierOperation{&updateOp}, OplogApplication::Mode::kInitialSync, @@ -1039,7 +1039,7 @@ TEST_F(OplogApplierImplTest, ImageCollectionInvalidationInInitialSyncHandlesConf // Apply the first oplog entry which should lead us to write an invalidate entry. { repl::UnreplicatedWritesBlock uwb(_opCtx.get()); - DisableDocumentValidation validationDisabler(_opCtx.get()); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx.get()); ASSERT_THROWS(applyOplogEntryOrGroupedInserts(_opCtx.get(), ApplierOperation{&invalidateOp}, OplogApplication::Mode::kInitialSync, @@ -1071,7 +1071,7 @@ TEST_F(OplogApplierImplTest, ImageCollectionInvalidationInInitialSyncHandlesConf { repl::UnreplicatedWritesBlock uwb(_opCtx.get()); - DisableDocumentValidation validationDisabler(_opCtx.get()); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx.get()); ASSERT_THROWS(applyOplogEntryOrGroupedInserts(_opCtx.get(), ApplierOperation{&earlierInvalidateOp}, OplogApplication::Mode::kInitialSync, @@ -3933,7 +3933,8 @@ TEST_F(OplogApplierImplTest, [&](OperationContext* opCtx, const NamespaceString&, const std::vector&) { onInsertsCalled = true; ASSERT_FALSE(opCtx->writesAreReplicated()); - ASSERT_TRUE(DocumentValidationSettings::get(opCtx).isSchemaValidationDisabled()); + ASSERT_TRUE( + DocumentValidationSettings::get(opCtx).isSchemaValidationDisabledForInternalOp()); }; createCollectionWithUuid(_opCtx.get(), nss); auto op = makeInsertDocumentOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss, BSON("_id" << 0)); diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp index 5a073020678..4ce1e028abb 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp @@ -274,7 +274,7 @@ Status OplogApplierImplTest::_applyOplogEntryOrGroupedInsertsWrapper( const OplogEntryOrGroupedInserts& batch, OplogApplication::Mode oplogApplicationMode) { UnreplicatedWritesBlock uwb(opCtx); - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); const bool dataIsConsistent = true; return applyOplogEntryOrGroupedInserts(opCtx, batch, oplogApplicationMode, dataIsConsistent); } @@ -295,7 +295,8 @@ void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( ASSERT_TRUE( shard_role_details::getLocker(opCtx)->isCollectionLockedForMode(targetNss, MODE_IX)); ASSERT_FALSE(opCtx->writesAreReplicated()); - ASSERT_TRUE(DocumentValidationSettings::get(opCtx).isSchemaValidationDisabled()); + ASSERT_TRUE( + DocumentValidationSettings::get(opCtx).isSchemaValidationDisabledForInternalOp()); }; _opObserver->onInsertsFn = diff --git a/src/mongo/db/repl/oplog_applier_utils.cpp b/src/mongo/db/repl/oplog_applier_utils.cpp index 92879aa01a9..e231081c555 100644 --- a/src/mongo/db/repl/oplog_applier_utils.cpp +++ b/src/mongo/db/repl/oplog_applier_utils.cpp @@ -426,7 +426,7 @@ Status OplogApplierUtils::applyOplogEntryOrGroupedInsertsCommon( const bool isDataConsistent, IncrementOpsAppliedStatsFn incrementOpsAppliedStats, OpCounters* opCounters) { - invariant(DocumentValidationSettings::get(opCtx).isSchemaValidationDisabled()); + invariant(DocumentValidationSettings::get(opCtx).isSchemaValidationDisabledForInternalOp()); const auto& op = entryOrGroupedInserts.getOp(); // Count each log op application as a separate operation, for reporting purposes. @@ -632,7 +632,7 @@ Status OplogApplierUtils::applyOplogBatchCommon( // We cannot do document validation, because document validation could have been disabled when // these oplog entries were generated. - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); // Group the operations by namespace in order to get larger groups for bulk inserts, but do not // mix up the current order of oplog entries within the same namespace (thus *stable* sort). stableSortByNamespace(ops); diff --git a/src/mongo/db/repl/oplog_buffer_collection.cpp b/src/mongo/db/repl/oplog_buffer_collection.cpp index 83cd92dbc50..249ede0f78f 100644 --- a/src/mongo/db/repl/oplog_buffer_collection.cpp +++ b/src/mongo/db/repl/oplog_buffer_collection.cpp @@ -223,7 +223,7 @@ void OplogBufferCollection::_push(WithLock, // (16MB user data + additional bytes for oplog fields like ’’op”, “ns”, “ui”). DisableDocumentValidation documentValidationDisabler( opCtx, - DocumentValidationSettings::kDisableSchemaValidation | + DocumentValidationSettings::kDisableSchemaValidationForInternalOp | DocumentValidationSettings::kDisableInternalValidation); write_ops::InsertCommandRequest insertOp(_nss); diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp index b82914c5464..98d5223566d 100644 --- a/src/mongo/db/repl/storage_interface_impl.cpp +++ b/src/mongo/db/repl/storage_interface_impl.cpp @@ -258,7 +258,7 @@ StorageInterfaceImpl::createCollectionForBulkLoading( // But, it's logically ok to disable internal validation as this function gets called // only during initial sync. DocumentValidationSettings::get(opCtx.get()) - .setFlags(DocumentValidationSettings::kDisableSchemaValidation | + .setFlags(DocumentValidationSettings::kDisableSchemaValidationForInternalOp | DocumentValidationSettings::kDisableInternalValidation); // Retry if WCE. diff --git a/src/mongo/db/repl/storage_interface_impl_test.cpp b/src/mongo/db/repl/storage_interface_impl_test.cpp index f33daaf892c..3d706b3e46d 100644 --- a/src/mongo/db/repl/storage_interface_impl_test.cpp +++ b/src/mongo/db/repl/storage_interface_impl_test.cpp @@ -242,12 +242,12 @@ private: _opCtx = cc().makeOperationContext(); // We are not replicating nor validating these writes. _uwb = std::make_unique(_opCtx.get()); - _ddv = std::make_unique(_opCtx.get()); + _ddv = std::make_unique(_opCtx.get()); } ServiceContext::UniqueOperationContext _opCtx; std::unique_ptr _uwb; - std::unique_ptr _ddv; + std::unique_ptr _ddv; ReplicationCoordinatorMock* _replicationCoordinatorMock = nullptr; }; diff --git a/src/mongo/db/repl/storage_timestamp_test.cpp b/src/mongo/db/repl/storage_timestamp_test.cpp index 0da1b7736c3..7552d32a547 100644 --- a/src/mongo/db/repl/storage_timestamp_test.cpp +++ b/src/mongo/db/repl/storage_timestamp_test.cpp @@ -881,7 +881,7 @@ TEST_F(StorageTimestampTest, SecondaryArrayInsertTimes) { // In order for oplog application to assign timestamps, we must be in non-replicated mode // and disable document validation. repl::UnreplicatedWritesBlock uwb(_opCtx); - DisableDocumentValidation validationDisabler(_opCtx); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx); // Create a new collection. NamespaceString nss = @@ -2789,7 +2789,7 @@ TEST_F(StorageTimestampTest, TimestampIndexOplogApplicationOnPrimary) { // In order for oplog application to assign timestamps, we must be in non-replicated mode // and disable document validation. repl::UnreplicatedWritesBlock uwb(_opCtx); - DisableDocumentValidation validationDisabler(_opCtx); + DisableDocumentValidationForInternalOp validationDisabler(_opCtx); std::string dbName = "unittest"; NamespaceString nss = diff --git a/src/mongo/db/repl/transaction_oplog_application.cpp b/src/mongo/db/repl/transaction_oplog_application.cpp index c906fcd9197..66c69e94fa0 100644 --- a/src/mongo/db/repl/transaction_oplog_application.cpp +++ b/src/mongo/db/repl/transaction_oplog_application.cpp @@ -749,7 +749,7 @@ void _reconstructPreparedTransaction(OperationContext* opCtx, repl::OplogApplication::Mode mode) { repl::UnreplicatedWritesBlock uwb(opCtx); // The transaction may have been prepared originally with document validation bypassed. - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); // The operations here are reconstructed at their prepare time. However, that time // will be ignored because there is an outer write unit of work during their diff --git a/src/mongo/db/s/migration_batch_inserter.cpp b/src/mongo/db/s/migration_batch_inserter.cpp index 3ecfe742a03..0adbdfa216e 100644 --- a/src/mongo/db/s/migration_batch_inserter.cpp +++ b/src/mongo/db/s/migration_batch_inserter.cpp @@ -189,7 +189,7 @@ void MigrationBatchInserter::run(Status status) const try { // and any internal validation for opCtx for performInserts() DisableDocumentValidation documentValidationDisabler( opCtx, - DocumentValidationSettings::kDisableSchemaValidation | + DocumentValidationSettings::kDisableSchemaValidationForInternalOp | DocumentValidationSettings::kDisableInternalValidation); const auto reply = write_ops_exec::performInserts( opCtx, insertOp, /*preConditions=*/boost::none, OperationSource::kFromMigrate); diff --git a/src/mongo/db/s/migration_chunk_cloner_source_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_test.cpp index 116f5e7a779..41dd1337134 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_test.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_test.cpp @@ -251,15 +251,10 @@ public: return _coll->parseValidator(opCtx, validator, allowedFeatures); } - void setValidator(OperationContext* opCtx, Validator validator) override { - MONGO_UNREACHABLE; - } - - Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) override { - MONGO_UNREACHABLE; - } - - Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) override { + Status setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 77918e16092..edfd76cedc5 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -659,7 +659,7 @@ repl::OpTime MigrationDestinationManager::fetchAndApplyBatch( while (true) { DisableDocumentValidation documentValidationDisabler( applicationOpCtx.get(), - DocumentValidationSettings::kDisableSchemaValidation | + DocumentValidationSettings::kDisableSchemaValidationForInternalOp | DocumentValidationSettings::kDisableInternalValidation); auto nextBatch = batches.pop(applicationOpCtx.get()); if (!applyBatchFn(applicationOpCtx.get(), nextBatch)) { diff --git a/src/mongo/db/server_feature_flags.idl b/src/mongo/db/server_feature_flags.idl index e801ca93f43..2a680837d87 100644 --- a/src/mongo/db/server_feature_flags.idl +++ b/src/mongo/db/server_feature_flags.idl @@ -118,6 +118,11 @@ feature_flags: default: true version: 8.1 fcv_gated: true + featureFlagValidatedValidationLevel: + description: "Enables the use of the validated schema validation level." + cpp_varname: gFeatureFlagValidatedValidationLevel + default: false + fcv_gated: true featureFlagDistinguishMetadataInconsistencyFromConflictingOperation: description: "Feature flag to distinguish ChunkMetadataInconsistency from ConflictingOperationInProgress error" cpp_varname: gDistinguishMetadataInconsistencyFromConflictingOperation diff --git a/src/mongo/db/shard_role/ddl/clone_catalog_data_command.cpp b/src/mongo/db/shard_role/ddl/clone_catalog_data_command.cpp index ac08b97e6b2..47c1d6d3dba 100644 --- a/src/mongo/db/shard_role/ddl/clone_catalog_data_command.cpp +++ b/src/mongo/db/shard_role/ddl/clone_catalog_data_command.cpp @@ -88,7 +88,7 @@ void cloneDatabase(OperationContext* opCtx, unsplittableCollections.end(), std::back_inserter(trackedColls)); - DisableDocumentValidation disableValidation(opCtx); + DisableDocumentValidationForInternalOp disableValidation(opCtx); // Clone the non-ignored collections. std::set clonedColls; diff --git a/src/mongo/db/shard_role/ddl/collmod_cmd.cpp b/src/mongo/db/shard_role/ddl/collmod_cmd.cpp index 9e87b00e46d..ad03a6a4fe8 100644 --- a/src/mongo/db/shard_role/ddl/collmod_cmd.cpp +++ b/src/mongo/db/shard_role/ddl/collmod_cmd.cpp @@ -165,14 +165,23 @@ public: str::stream() << "Document validators not allowed on system collection " << nss.toStringForErrorMsg(), nss != NamespaceString::kConfigSettingsNamespace); - } - if (cmd.getValidationAction() == ValidationActionEnum::errorAndLog) { - uassert( - ErrorCodes::InvalidOptions, - "Validation action 'errorAndLog' is not supported with current FCV", - gFeatureFlagErrorAndLogValidationAction.isEnabledUseLastLTSFCVWhenUninitialized( - VersionContext::getDecoration(opCtx), - serverGlobalParams.featureCompatibility.acquireFCVSnapshot())); + + const auto fcvSnapshot = + serverGlobalParams.featureCompatibility.acquireFCVSnapshot(); + if (cmd.getValidationAction() == ValidationActionEnum::errorAndLog) { + uassert(ErrorCodes::InvalidOptions, + "Validation action 'errorAndLog' is not supported with current FCV", + gFeatureFlagErrorAndLogValidationAction + .isEnabledUseLastLTSFCVWhenUninitialized( + VersionContext::getDecoration(opCtx), fcvSnapshot)); + } + if (cmd.getValidationLevel() == ValidationLevelEnum::validated) { + uassert(ErrorCodes::InvalidOptions, + "Validation level 'validated' is not supported with current FCV", + gFeatureFlagValidatedValidationLevel + .isEnabledUseLastLTSFCVWhenUninitialized( + VersionContext::getDecoration(opCtx), fcvSnapshot)); + } } // We do not use the serialization context for reply object serialization as the reply diff --git a/src/mongo/db/shard_role/ddl/create.idl b/src/mongo/db/shard_role/ddl/create.idl index 2007731471f..7923a304df5 100644 --- a/src/mongo/db/shard_role/ddl/create.idl +++ b/src/mongo/db/shard_role/ddl/create.idl @@ -104,7 +104,7 @@ structs: description: "Determines how strictly to apply the validation rules to existing documents during an update. - Can be one of following values: 'off', 'strict' or 'moderate'." + Can be one of following values: 'off', 'strict', 'moderate' or 'validated'." type: ValidationLevel optional: true stability: stable diff --git a/src/mongo/db/shard_role/ddl/create_command.cpp b/src/mongo/db/shard_role/ddl/create_command.cpp index 01ca4ea5a90..e7284fe90b5 100644 --- a/src/mongo/db/shard_role/ddl/create_command.cpp +++ b/src/mongo/db/shard_role/ddl/create_command.cpp @@ -523,14 +523,23 @@ public: str::stream() << "Document validators not allowed on system collection " << ns().toStringForErrorMsg(), ns() != NamespaceString::kConfigSettingsNamespace); - } - if (cmd.getValidationAction() == ValidationActionEnum::errorAndLog) { - uassert( - ErrorCodes::InvalidOptions, - "Validation action 'errorAndLog' is not supported with current FCV", - gFeatureFlagErrorAndLogValidationAction.isEnabledUseLastLTSFCVWhenUninitialized( - VersionContext::getDecoration(opCtx), - serverGlobalParams.featureCompatibility.acquireFCVSnapshot())); + + const auto fcvSnapshot = + serverGlobalParams.featureCompatibility.acquireFCVSnapshot(); + if (cmd.getValidationAction() == ValidationActionEnum::errorAndLog) { + uassert(ErrorCodes::InvalidOptions, + "Validation action 'errorAndLog' is not supported with current FCV", + gFeatureFlagErrorAndLogValidationAction + .isEnabledUseLastLTSFCVWhenUninitialized( + VersionContext::getDecoration(opCtx), fcvSnapshot)); + } + if (cmd.getValidationLevel() == ValidationLevelEnum::validated) { + uassert(ErrorCodes::InvalidOptions, + "Validation level 'validated' is not supported with current FCV", + gFeatureFlagValidatedValidationLevel + .isEnabledUseLastLTSFCVWhenUninitialized( + VersionContext::getDecoration(opCtx), fcvSnapshot)); + } } OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE diff --git a/src/mongo/db/shard_role/shard_catalog/BUILD.bazel b/src/mongo/db/shard_role/shard_catalog/BUILD.bazel index 0763f5eb1b4..31e5bb0a875 100644 --- a/src/mongo/db/shard_role/shard_catalog/BUILD.bazel +++ b/src/mongo/db/shard_role/shard_catalog/BUILD.bazel @@ -167,6 +167,7 @@ mongo_cc_library( name = "document_validation", srcs = [ "document_validation.cpp", + "document_validation_helpers.cpp", ], deps = [ "//src/mongo/db:service_context", diff --git a/src/mongo/db/shard_role/shard_catalog/coll_mod.cpp b/src/mongo/db/shard_role/shard_catalog/coll_mod.cpp index 1baaa4998c2..06ace25c2df 100644 --- a/src/mongo/db/shard_role/shard_catalog/coll_mod.cpp +++ b/src/mongo/db/shard_role/shard_catalog/coll_mod.cpp @@ -1011,17 +1011,13 @@ Status _collModInternal(OperationContext* opCtx, processCollModIndexRequest( opCtx, writableColl, cmrNew.indexRequest, &indexCollModInfo, result, mode); - if (cmrNew.collValidator) { - writableColl->setValidator(opCtx, *cmrNew.collValidator); - } - if (cmrNew.collValidationAction) + if (cmrNew.collValidationLevel || cmrNew.collValidationAction || cmrNew.collValidator) { uassertStatusOKWithContext( - writableColl->setValidationAction(opCtx, *cmrNew.collValidationAction), - "Failed to set validationAction"); - if (cmrNew.collValidationLevel) { - uassertStatusOKWithContext( - writableColl->setValidationLevel(opCtx, *cmrNew.collValidationLevel), - "Failed to set validationLevel"); + writableColl->setValidationOptions(opCtx, + cmrNew.collValidationLevel, + cmrNew.collValidationAction, + cmrNew.collValidator), + "Failed to set validation options"); } if (cmrNew.changeStreamPreAndPostImagesOptions.has_value() && diff --git a/src/mongo/db/shard_role/shard_catalog/collection.h b/src/mongo/db/shard_role/shard_catalog/collection.h index f0e4c1af8d2..d020befe765 100644 --- a/src/mongo/db/shard_role/shard_catalog/collection.h +++ b/src/mongo/db/shard_role/shard_catalog/collection.h @@ -367,15 +367,15 @@ public: MatchExpressionParser::AllowedFeatureSet allowedFeatures) const = 0; /** - * Sets the validator for this collection. + * Sets the validation options for this collection. * - * An empty validator removes all validation. - * Requires an exclusive lock on the collection. + * A boost::none parameter means that it should not be changed or that the default will be + * used. */ - virtual void setValidator(OperationContext* opCtx, Validator validator) = 0; - - virtual Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) = 0; - virtual Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) = 0; + virtual Status setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) = 0; virtual boost::optional getValidationLevel() const = 0; virtual boost::optional getValidationAction() const = 0; @@ -986,11 +986,32 @@ inline ValidationActionEnum validationActionOrDefault( return action.value_or(ValidationActionEnum::error); } +MONGO_MOD_PUBLIC +inline ValidationActionEnum validationActionOrCurrent( + const CollectionOptions& opts, boost::optional action) { + return action.value_or(validationActionOrDefault(opts.validationAction)); +} + MONGO_MOD_PUBLIC inline ValidationLevelEnum validationLevelOrDefault(boost::optional level) { return level.value_or(ValidationLevelEnum::strict); } +MONGO_MOD_PUBLIC +inline ValidationLevelEnum validationLevelOrCurrent(const CollectionOptions& opts, + boost::optional level) { + return level.value_or(validationLevelOrDefault(opts.validationLevel)); +} + +/** + * Mandatory level means that the schema validator is strictly enforced on all inserts and updates. + * 'validated' and 'strict' both forbid inserting/updating non-conforming documents. + */ +MONGO_MOD_PUBLIC +inline bool validationLevelIsMandatory(ValidationLevelEnum level) { + return level == ValidationLevelEnum::strict || level == ValidationLevelEnum::validated; +} + /** * Returns a collator for the user-specified collation 'userCollation'. * diff --git a/src/mongo/db/shard_role/shard_catalog/collection_impl.cpp b/src/mongo/db/shard_role/shard_catalog/collection_impl.cpp index 3a5c53ba7a0..95c54fcfdb5 100644 --- a/src/mongo/db/shard_role/shard_catalog/collection_impl.cpp +++ b/src/mongo/db/shard_role/shard_catalog/collection_impl.cpp @@ -87,6 +87,7 @@ #include "mongo/db/shard_role/shard_catalog/catalog_stats.h" #include "mongo/db/shard_role/shard_catalog/collection_catalog.h" #include "mongo/db/shard_role/shard_catalog/document_validation.h" +#include "mongo/db/shard_role/shard_catalog/document_validation_helpers.h" #include "mongo/db/shard_role/shard_catalog/durable_catalog.h" #include "mongo/db/shard_role/shard_catalog/index_catalog_impl.h" #include "mongo/db/shard_role/shard_catalog/index_descriptor.h" @@ -126,26 +127,6 @@ MONGO_FAIL_POINT_DEFINE(skipCappedDeletes); // and clear the new durable flag which is stored inside the collection options. MONGO_FAIL_POINT_DEFINE(simulateLegacyTimeseriesMixedSchemaFlag); -Status checkValidationOptionsCanBeUsed(const CollectionOptions& opts, - boost::optional newLevel, - boost::optional newAction) { - if (!opts.encryptedFieldConfig) { - return Status::OK(); - } - if (validationLevelOrDefault(newLevel) != ValidationLevelEnum::strict) { - return Status( - ErrorCodes::BadValue, - "Validation levels other than 'strict' are not allowed on encrypted collections"); - } - auto action = validationActionOrDefault(newAction); - if (action == ValidationActionEnum::warn || action == ValidationActionEnum::errorAndLog) { - return Status(ErrorCodes::BadValue, - "Validation action of 'warn' and 'errorAndLog' are not allowed on encrypted " - "collections"); - } - return Status::OK(); -} - /** * Returns true if we are running retryable write or retryable internal multi-document transaction. */ @@ -435,8 +416,8 @@ void CollectionImpl::_initCommon(OperationContext* opCtx) { uassertStatusOK(_checkValidatorCanBeUsed(validatorDoc)); // Make sure validationAction and validationLevel are allowed on this collection - uassertStatusOK(checkValidationOptionsCanBeUsed( - collectionOptions, collectionOptions.validationLevel, collectionOptions.validationAction)); + uassertStatusOK( + checkValidationOptionsCanBeUsed(collectionOptions, boost::none, boost::none, boost::none)); // Make sure to copy the action and level before parsing MatchExpression, since certain features // are not supported with certain combinations of action and level. @@ -594,8 +575,19 @@ std::pair CollectionImpl::checkValid if (validationLevelOrDefault(_metadata->options.validationLevel) == ValidationLevelEnum::off) return {SchemaValidationResult::kPass, Status::OK()}; - if (DocumentValidationSettings::get(opCtx).isSchemaValidationDisabled()) + if (DocumentValidationSettings::get(opCtx).isSchemaValidationDisabledForInternalOp()) { return {SchemaValidationResult::kPass, Status::OK()}; + } + + if (DocumentValidationSettings::get(opCtx).isSchemaValidationDisabled()) { + if (_metadata->options.validationLevel == ValidationLevelEnum::validated) { + return {SchemaValidationResult::kError, + Status(ErrorCodes::BadValue, + "bypassDocumentValidation is not permitted on 'validated' collections")}; + } else { + return {SchemaValidationResult::kPass, Status::OK()}; + } + } if (ns().isTemporaryReshardingCollection()) { // In resharding, the donor shard primary is responsible for performing document validation @@ -623,6 +615,11 @@ std::pair CollectionImpl::checkValid switch (validationActionOrDefault(_metadata->options.validationAction)) { case ValidationActionEnum::warn: + if (validationLevelOrDefault(_metadata->options.validationLevel) == + ValidationLevelEnum::validated) { + // Warn is prohibited for validated collections + return {SchemaValidationResult::kError, status}; + } return {SchemaValidationResult::kWarn, status}; case ValidationActionEnum::error: return {SchemaValidationResult::kError, status}; @@ -1210,20 +1207,37 @@ Status CollectionImpl::truncate(OperationContext* opCtx) { return Status::OK(); } -void CollectionImpl::setValidator(OperationContext* opCtx, Validator validator) { +Status CollectionImpl::setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) { invariant(shard_role_details::getLocker(opCtx)->isCollectionLockedForMode(ns(), MODE_X)); + auto status = + checkValidationOptionsCanBeUsed(_metadata->options, newLevel, newAction, newValidator); + if (!status.isOK()) { + return status; + } - auto validatorDoc = validator.validatorDoc.getOwned(); - auto validationLevel = validationLevelOrDefault(_metadata->options.validationLevel); - auto validationAction = validationActionOrDefault(_metadata->options.validationAction); + auto validationLevel = validationLevelOrCurrent(_metadata->options, newLevel); + auto validationAction = validationActionOrCurrent(_metadata->options, newAction); + if (newValidator) { + _validator = std::move(*newValidator); + } + + if (auto [mustReparse, allowedFeatures] = mustReparseValidator(newLevel, newAction); + mustReparse) { + _validator = parseValidator(opCtx, _validator.validatorDoc, allowedFeatures); + if (!_validator.isOK()) { + return _validator.getStatus(); + } + } _writeMetadata(opCtx, [&](durable_catalog::CatalogEntryMetaData& md) { - md.options.validator = validatorDoc; + md.options.validator = _validator.validatorDoc; md.options.validationLevel = validationLevel; md.options.validationAction = validationAction; }); - - _validator = std::move(validator); + return Status::OK(); } boost::optional CollectionImpl::getValidationLevel() const { @@ -1234,74 +1248,17 @@ boost::optional CollectionImpl::getValidationAction() cons return _metadata->options.validationAction; } -Status CollectionImpl::setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) { - invariant(shard_role_details::getLocker(opCtx)->isCollectionLockedForMode(ns(), MODE_X)); - - auto status = checkValidationOptionsCanBeUsed(_metadata->options, newLevel, boost::none); - if (!status.isOK()) { - return status; - } - - auto storedValidationLevel = validationLevelOrDefault(newLevel); - - // Reparse the validator as there are some features which are only supported with certain - // validation levels. - auto allowedFeatures = MatchExpressionParser::kAllowAllSpecialFeatures; - if (storedValidationLevel == ValidationLevelEnum::moderate) - allowedFeatures &= ~MatchExpressionParser::AllowedFeatures::kEncryptKeywords; - - _validator = parseValidator(opCtx, _validator.validatorDoc, allowedFeatures); - if (!_validator.isOK()) { - return _validator.getStatus(); - } - - _writeMetadata(opCtx, [&](durable_catalog::CatalogEntryMetaData& md) { - md.options.validator = _validator.validatorDoc; - md.options.validationLevel = storedValidationLevel; - md.options.validationAction = validationActionOrDefault(md.options.validationAction); - }); - - return Status::OK(); -} - -Status CollectionImpl::setValidationAction(OperationContext* opCtx, - ValidationActionEnum newAction) { - invariant(shard_role_details::getLocker(opCtx)->isCollectionLockedForMode(ns(), MODE_X)); - - auto status = checkValidationOptionsCanBeUsed(_metadata->options, boost::none, newAction); - if (!status.isOK()) { - return status; - } - - auto storedValidationAction = validationActionOrDefault(newAction); - - // Reparse the validator as there are some features which are only supported with certain - // validation actions. - auto allowedFeatures = MatchExpressionParser::kAllowAllSpecialFeatures; - if (storedValidationAction == ValidationActionEnum::warn || - storedValidationAction == ValidationActionEnum::errorAndLog) - allowedFeatures &= ~MatchExpressionParser::AllowedFeatures::kEncryptKeywords; - - _validator = parseValidator(opCtx, _validator.validatorDoc, allowedFeatures); - if (!_validator.isOK()) { - return _validator.getStatus(); - } - - _writeMetadata(opCtx, [&](durable_catalog::CatalogEntryMetaData& md) { - md.options.validator = _validator.validatorDoc; - md.options.validationLevel = validationLevelOrDefault(md.options.validationLevel); - md.options.validationAction = storedValidationAction; - }); - - return Status::OK(); -} - Status CollectionImpl::updateValidator(OperationContext* opCtx, BSONObj newValidatorDoc, boost::optional newLevel, boost::optional newAction) { invariant(shard_role_details::getLocker(opCtx)->isCollectionLockedForMode(ns(), MODE_X)); + auto validationLevel = validationLevelOrCurrent(_metadata->options, newLevel); + if (validationLevel == ValidationLevelEnum::validated) { + return Status(ErrorCodes::BadValue, + "Validator can not be changed on 'validated' collections"); + } tassert(11738200, fmt::format("Illegal attempt to set a non-empty validator on viewless timeseries " "collection '{}'", @@ -1309,17 +1266,18 @@ Status CollectionImpl::updateValidator(OperationContext* opCtx, !isTimeseriesCollection() || !isNewTimeseriesWithoutView() || (newValidatorDoc.isEmpty() && !newLevel.has_value() && !newAction.has_value())); - auto status = checkValidationOptionsCanBeUsed(_metadata->options, newLevel, newAction); - if (!status.isOK()) { - return status; - } - auto newValidator = parseValidator(opCtx, newValidatorDoc, MatchExpressionParser::kAllowAllSpecialFeatures); if (!newValidator.isOK()) { return newValidator.getStatus(); } + if (auto status = + checkValidationOptionsCanBeUsed(_metadata->options, newLevel, newAction, newValidator); + !status.isOK()) { + return status; + } + _writeMetadata(opCtx, [&](durable_catalog::CatalogEntryMetaData& md) { md.options.validator = newValidatorDoc; md.options.validationLevel = newLevel; diff --git a/src/mongo/db/shard_role/shard_catalog/collection_impl.h b/src/mongo/db/shard_role/shard_catalog/collection_impl.h index 5fa71f0f950..234eaa3f8f7 100644 --- a/src/mongo/db/shard_role/shard_catalog/collection_impl.h +++ b/src/mongo/db/shard_role/shard_catalog/collection_impl.h @@ -191,13 +191,15 @@ public: /** * Sets the validator for this collection. * - * An empty validator removes all validation. + * A boost::empty parameter means that it should not be changed or that the default will be + * used. * Requires an exclusive lock on the collection. */ - void setValidator(OperationContext* opCtx, Validator validator) final; + Status setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) final; - Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) final; - Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) final; boost::optional getValidationLevel() const final; boost::optional getValidationAction() const final; diff --git a/src/mongo/db/shard_role/shard_catalog/collection_mock.h b/src/mongo/db/shard_role/shard_catalog/collection_mock.h index b094a13fdb3..d55dce3f670 100644 --- a/src/mongo/db/shard_role/shard_catalog/collection_mock.h +++ b/src/mongo/db/shard_role/shard_catalog/collection_mock.h @@ -160,14 +160,10 @@ public: MONGO_UNREACHABLE; } - void setValidator(OperationContext* opCtx, Validator validator) override { - MONGO_UNREACHABLE; - } - - Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) override { - MONGO_UNREACHABLE; - } - Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) override { + Status setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/shard_role/shard_catalog/collection_options.idl b/src/mongo/db/shard_role/shard_catalog/collection_options.idl index b2255f06666..5fcfd2c1fed 100644 --- a/src/mongo/db/shard_role/shard_catalog/collection_options.idl +++ b/src/mongo/db/shard_role/shard_catalog/collection_options.idl @@ -43,6 +43,7 @@ enums: "off": "off" strict: strict moderate: moderate + validated: validated ValidationAction: description: "Determines an action on invalid documents being written: diff --git a/src/mongo/db/shard_role/shard_catalog/document_validation.h b/src/mongo/db/shard_role/shard_catalog/document_validation.h index 880a8e1b22f..933ab19b1d8 100644 --- a/src/mongo/db/shard_role/shard_catalog/document_validation.h +++ b/src/mongo/db/shard_role/shard_catalog/document_validation.h @@ -67,23 +67,29 @@ public: */ kEnableValidation = 0x00, /* - * Disables the schema validation during document inserts and updates. - * This flag should be enabled if WriteCommandRequestBase::_bypassDocumentValidation - * is set to true. + * Disables the schema validation during document inserts and updates for user-initiated + * operations. This flag should be enabled if + * WriteCommandRequestBase::_bypassDocumentValidation is set to true. */ - kDisableSchemaValidation = 0x01, + kDisableSchemaValidationRequestedByUser = 0x01, + /* + * Disables the schema validation during all document inserts and updates including internal + * operations such as oplog application and initial sync. This flag should only be set for + * internal operations. + */ + kDisableSchemaValidationForInternalOp = 0x02, /* * Disables any internal validation (like fixDocumentForInsert()). This flag * should be enabled only for trusted internal writes or internal writes that * doesn't comply with internal validation rules. */ - kDisableInternalValidation = 0x02, + kDisableInternalValidation = 0x04, /* * If set, modifications to the safeContent array are allowed. This flag is only * enabled when bypass document validation is enabled or if crudProcessed is true * in the query. */ - kDisableSafeContentValidation = 0x04, + kDisableSafeContentValidation = 0x08, }; using Flags = std::uint8_t; @@ -102,7 +108,12 @@ public: } bool isSchemaValidationDisabled() const { - return _flags & kDisableSchemaValidation; + return _flags & + (kDisableSchemaValidationRequestedByUser | kDisableSchemaValidationForInternalOp); + } + + bool isSchemaValidationDisabledForInternalOp() const { + return _flags & kDisableSchemaValidationForInternalOp; } bool isInternalValidationDisabled() const { @@ -130,9 +141,7 @@ class MONGO_MOD_NEEDS_REPLACEMENT DisableDocumentValidation { DisableDocumentValidation& operator=(const DisableDocumentValidation&) = delete; public: - DisableDocumentValidation(OperationContext* opCtx, - DocumentValidationSettings::Flags flags = - DocumentValidationSettings::kDisableSchemaValidation) + DisableDocumentValidation(OperationContext* opCtx, DocumentValidationSettings::Flags flags) : _opCtx(opCtx) { auto& documentValidationSettings = DocumentValidationSettings::get(_opCtx); _initialState = documentValidationSettings; @@ -148,16 +157,27 @@ private: DocumentValidationSettings _initialState; }; -/** - * Disables document schema validation while in scope if the constructor is passed true. - */ -class MONGO_MOD_NEEDS_REPLACEMENT DisableDocumentSchemaValidationIfTrue { +class MONGO_MOD_NEEDS_REPLACEMENT DisableDocumentValidationForInternalOp { public: - DisableDocumentSchemaValidationIfTrue(OperationContext* opCtx, - bool shouldDisableSchemaValidation) { + DisableDocumentValidationForInternalOp(OperationContext* opCtx) + : _documentSchemaValidationDisabler( + opCtx, DocumentValidationSettings::kDisableSchemaValidationForInternalOp) {} + +private: + DisableDocumentValidation _documentSchemaValidationDisabler; +}; + +/** + * Disables document schema validation for user requests while in scope if the constructor is passed + * true. + */ +class MONGO_MOD_NEEDS_REPLACEMENT DisableDocumentSchemaValidationRequestedByUserIfTrue { +public: + DisableDocumentSchemaValidationRequestedByUserIfTrue(OperationContext* opCtx, + bool shouldDisableSchemaValidation) { if (shouldDisableSchemaValidation) { _documentSchemaValidationDisabler.emplace( - opCtx, DocumentValidationSettings::kDisableSchemaValidation); + opCtx, DocumentValidationSettings::kDisableSchemaValidationRequestedByUser); } } diff --git a/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.cpp b/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.cpp new file mode 100644 index 00000000000..9f8e901adad --- /dev/null +++ b/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.cpp @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2026-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/shard_role/shard_catalog/document_validation_helpers.h" + +namespace mongo { + +Status checkValidationOptionsCanBeUsed(const CollectionOptions& opts, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) { + + auto validationAction = validationActionOrCurrent(opts, newAction); + auto validationLevel = validationLevelOrCurrent(opts, newLevel); + if (opts.encryptedFieldConfig) { + if (!validationLevelIsMandatory(validationLevel)) { + return Status(ErrorCodes::BadValue, + "Validation levels other than 'strict' or 'validated' are not allowed on " + "encrypted collections"); + } + if (validationAction == ValidationActionEnum::warn || + validationAction == ValidationActionEnum::errorAndLog) { + return Status( + ErrorCodes::BadValue, + "Validation action of 'warn' and 'errorAndLog' are not allowed on encrypted " + "collections"); + } + } + if (validationLevel == ValidationLevelEnum::validated) { + if (validationAction == ValidationActionEnum::warn) { + return Status( + ErrorCodes::BadValue, + "Validation action of 'warn' is not allowed when Validation level is 'validated'"); + } + if (opts.uuid) { // existing collection + if (opts.validationLevel != ValidationLevelEnum::validated) { + return Status( + ErrorCodes::BadValue, + "Validation level 'validated' can not be set on existing collections."); + } + if (newValidator) { + return Status(ErrorCodes::BadValue, + "Validator can not be changed when Validation level is 'validated'"); + } + } + } + return Status::OK(); +} + +std::pair mustReparseValidator( + boost::optional newLevel, + boost::optional newAction) { + + auto allowedFeatures = MatchExpressionParser::kAllowAllSpecialFeatures; + + if (!newAction && !newLevel) { + return {false, allowedFeatures}; + } + if (newLevel == ValidationLevelEnum::moderate || newAction == ValidationActionEnum::warn || + newAction == ValidationActionEnum::errorAndLog) { + allowedFeatures &= ~MatchExpressionParser::AllowedFeatures::kEncryptKeywords; + } + return {true, allowedFeatures}; +} + +} // namespace mongo diff --git a/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.h b/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.h new file mode 100644 index 00000000000..ca1c91a048c --- /dev/null +++ b/src/mongo/db/shard_role/shard_catalog/document_validation_helpers.h @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2026-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/shard_role/shard_catalog/collection.h" +#include "mongo/db/shard_role/shard_catalog/collection_options.h" + + +namespace mongo { +MONGO_MOD_PRIVATE Status +checkValidationOptionsCanBeUsed(const CollectionOptions& opts, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator); + +MONGO_MOD_PRIVATE std::pair mustReparseValidator( + boost::optional newLevel, boost::optional newAction); + +} // namespace mongo diff --git a/src/mongo/db/shard_role/shard_catalog/rename_collection.cpp b/src/mongo/db/shard_role/shard_catalog/rename_collection.cpp index 2b63004339a..5e4d5fe53ad 100644 --- a/src/mongo/db/shard_role/shard_catalog/rename_collection.cpp +++ b/src/mongo/db/shard_role/shard_catalog/rename_collection.cpp @@ -347,7 +347,7 @@ Status renameCollectionWithinDB(OperationContext* opCtx, const NamespaceString& target, RenameCollectionOptions options) { invariant(source.isEqualDb(target)); - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); CollectionOrViewAcquisitionRequests acquisitionRequests = { CollectionOrViewAcquisitionRequest::fromOpCtx( @@ -516,7 +516,7 @@ Status renameCollectionWithinDBForApplyOps(OperationContext* opCtx, repl::OpTime renameOpTimeFromApplyOps, const RenameCollectionOptions& options) { invariant(source.isEqualDb(target)); - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); AutoGetDb autoDb(opCtx, source.dbName(), MODE_IX); auto acqStatus = @@ -874,7 +874,7 @@ Status renameCollectionAcrossDatabases(OperationContext* opCtx, return acqStatus.getStatus(); auto& [sourceColl, tmpName, tempCollLock] = acqStatus.getValue(); - DisableDocumentValidation validationDisabler(opCtx); + DisableDocumentValidationForInternalOp validationDisabler(opCtx); if (!sourceDB.getDb()) return Status(ErrorCodes::NamespaceNotFound, "source namespace does not exist"); diff --git a/src/mongo/db/shard_role/shard_catalog/virtual_collection_impl.h b/src/mongo/db/shard_role/shard_catalog/virtual_collection_impl.h index 50417e68300..74e9eb0ee16 100644 --- a/src/mongo/db/shard_role/shard_catalog/virtual_collection_impl.h +++ b/src/mongo/db/shard_role/shard_catalog/virtual_collection_impl.h @@ -212,16 +212,11 @@ public: return Validator(); } - void setValidator(OperationContext* opCtx, Validator validator) final { - unimplementedTasserted(); - } - Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) final { - unimplementedTasserted(); - return Status(ErrorCodes::UnknownError, "unknown"); - } - - Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) final { + Status setValidationOptions(OperationContext* opCtx, + boost::optional newLevel, + boost::optional newAction, + boost::optional newValidator) final { unimplementedTasserted(); return Status(ErrorCodes::UnknownError, "unknown"); } diff --git a/src/mongo/db/timeseries/timeseries_write_util.cpp b/src/mongo/db/timeseries/timeseries_write_util.cpp index 55f04daf37e..e9e90439849 100644 --- a/src/mongo/db/timeseries/timeseries_write_util.cpp +++ b/src/mongo/db/timeseries/timeseries_write_util.cpp @@ -198,7 +198,7 @@ void performAtomicWrites( 7655102, "must specify at least one type of write", modificationOp || !insertOps.empty()); NamespaceString ns = coll->ns(); - DisableDocumentValidation disableDocumentValidation{opCtx}; + DisableDocumentValidationForInternalOp disableDocumentValidation{opCtx}; write_ops_exec::LastOpFixer lastOpFixer{opCtx}; lastOpFixer.startingOp(ns); diff --git a/src/mongo/db/timeseries/write_ops/internal/timeseries_write_ops_internal.cpp b/src/mongo/db/timeseries/write_ops/internal/timeseries_write_ops_internal.cpp index 05b964608ee..a35226ed7a8 100644 --- a/src/mongo/db/timeseries/write_ops/internal/timeseries_write_ops_internal.cpp +++ b/src/mongo/db/timeseries/write_ops/internal/timeseries_write_ops_internal.cpp @@ -930,7 +930,7 @@ Status performAtomicTimeseriesWrites( auto originalNss = !insertOps.empty() ? insertOps.front().getNamespace() : updateOps.front().getNamespace(); - DisableDocumentValidation disableDocumentValidation{opCtx}; + DisableDocumentValidationForInternalOp disableDocumentValidation{opCtx}; write_ops_exec::LastOpFixer lastOpFixer(opCtx); lastOpFixer.startingOp(originalNss);