mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-91746 Cleanup deprecated index metadata fields on FCV upgrade (#32255)

Browse Source 
GitOrigin-RevId: fad271d80afcd43532e91eab7f1a7b5cc6e88331
This commit is contained in:
Yujin Kang Park 2025-04-01 18:12:59 +02:00 committed by MongoDB Bot
parent ede97c6e9d
commit 31ccfb3c01
20 changed files with 424 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eslint.config.mjs
View File

@ -148,8 +148,10 @@ export default [
copyDir: true,
copyFile: true,
decompressBSONColumn: true,
dumpBSONAsHex: true,
getFileMode: true,
getStringWidth: true,
hexToBSON: true,
passwordPrompt: true,
umask: true,
writeFile: true,

39
jstests/disk/libs/wt_file_helper.js
View File

@ -422,3 +422,42 @@ export let insertInvalidUTF8 = function(coll, uri, conn, numDocs) {
};
rewriteTable(uri, conn, makeInvalidUTF8);
};
/**
* Loads the contents of the _mdb_catalog.wt file, and invokes the 'modifyCatalogEntry' callback for
* each entry, which should modify the passed entry (in-place). The modified contents are then
* written back to the _mdb_catalog.wt file.
*/
export let rewriteCatalogTable = function(conn, modifyCatalogEntry) {
const uri = "_mdb_catalog";
const fullURI = "table:" + uri;
const separator = _isWindows() ? '\\' : '/';
const tempDumpFile = conn.dbpath + separator + "temp_dump";
const newTableFile = conn.dbpath + separator + "new_table_file" + count++;
runWiredTigerTool("-h",
conn.dbpath,
"-r",
"-C",
"log=(compressor=snappy,path=journal)",
"dump",
"-x",
"-f",
tempDumpFile,
fullURI);
let lines = cat(tempDumpFile).split("\n");
// Each record takes two lines with a key and a value. We will only modify the values.
for (let i = wtHeaderLines; i < lines.length; i += 2) {
let entry = hexToBSON(lines[i]);
modifyCatalogEntry(entry);
lines[i] = dumpBSONAsHex(entry);
}
writeFile(newTableFile, lines.join("\n"));
runWiredTigerTool("-h", conn.dbpath, "alter", fullURI, "write_timestamp_usage=never");
runWiredTigerTool("-h", conn.dbpath, "load", "-f", newTableFile, "-r", uri);
runWiredTigerTool("-h", conn.dbpath, "alter", fullURI, "write_timestamp_usage=none");
};

170
jstests/multiVersion/targetedTestsLastLtsFeatures/cleanup_index_spec_on_upgrade.js Normal file
View File

@ -0,0 +1,170 @@
/**
* Test index specs in an old format are cleaned up on FCV transition to 9.0.
*/
import "jstests/multiVersion/libs/multi_rs.js";
import {rewriteCatalogTable} from "jstests/disk/libs/wt_file_helper.js";
import {ReplSetTest} from "jstests/libs/replsettest.js";
import {ShardingTest} from "jstests/libs/shardingtest.js";
const databaseName = jsTestName();
const collectionName = "coll";
// Rewrites the durable catalog for the given namespace, injecting the "ns" field to the
// "md.indexes.spec" object, to simulate 4.2 metadata.
function rewriteNodeCatalog(rst, node, namespace) {
jsTestLog("Rewriting durable catalog in: " + node.dbpath);
rst.stop(node, /*signal=*/ null, /*opts=*/ null, {forRestart: true, waitpid: true});
function durableCatalogModFn(entry) {
if (entry.ns == namespace) {
entry.md.indexes.forEach((index) => {
index.spec.ns = namespace;
});
}
}
rewriteCatalogTable(node, durableCatalogModFn);
rst.restart(node);
}
function assertExpectedCollectionIndexSpecWithNamespaceField(node, expected) {
let nodeColl = node.getDB(databaseName)[collectionName];
let listCatalogRes = nodeColl.aggregate([{$listCatalog: {}}]).toArray();
let filterRes = listCatalogRes.filter((entry) => entry.md.ns == nodeColl.getFullName());
assert.eq(filterRes.length, 1);
let collEntry = filterRes[0];
function errMsg() {
if (expected)
return `Node: ${node.host}:${
node.port}. Expected 'ns' field to be found in index spec in : ${
tojson(collEntry)}`;
else
return `Node: ${node.host}:${
node.port}. Expected 'ns' field to NOT be found in index spec in : ${
tojson(collEntry)}`;
}
collEntry.md.indexes.forEach((index) => {
assert.eq(index.spec.hasOwnProperty("ns"), expected, errMsg);
});
}
function rewriteReplicaSetCatalog(rst, fullCollName) {
rst.awaitReplication();
// Rewrite durable catalog on all replicas.
rst.nodes.forEach((node) => {
if (node == rst.getPrimary()) {
rst.stepUp(rst.getSecondary());
}
// This implicitly tests startup behavior with an old data format. The namespace field must
// remain a valid index spec field until we are certain (post 9.0) that it is no longer
// possible for it to be present.
rewriteNodeCatalog(rst, node, fullCollName);
});
rst.awaitSecondaryNodes();
}
function testForReplicaSet() {
const rst = new ReplSetTest({nodes: 2, nodeOptions: {binVersion: 'latest'}});
rst.startSet();
rst.initiate();
let db = rst.getPrimary().getDB(databaseName);
let coll = db[collectionName];
// Ensure we are in last LTS FCV (8.0).
assert.commandWorked(
rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true}));
assert.commandWorked(db.runCommand({
createIndexes: coll.getName(),
indexes: [{key: {a: 1}, name: 'a_1'}],
}));
rewriteReplicaSetCatalog(rst, coll.getFullName());
// Check metadata is in 4.2 format.
rst.nodes.forEach((node) => {
assertExpectedCollectionIndexSpecWithNamespaceField(node, true);
});
assert.commandWorked(
rst.getPrimary().adminCommand({setFeatureCompatibilityVersion: latestFCV, confirm: true}));
// Check metadata has been cleaned up.
rst.nodes.forEach((node) => {
assertExpectedCollectionIndexSpecWithNamespaceField(node, false);
});
rst.stopSet();
}
function testForShardedCluster() {
const st = new ShardingTest({shards: 2, mongos: 1, config: 1, rs: {nodes: 2}});
// Ensure we are in last LTS FCV (8.0).
assert.commandWorked(
st.s.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV, confirm: true}));
const fullCollName = databaseName + "." + collectionName;
assert.commandWorked(
st.s.adminCommand({enableSharding: databaseName, primaryShard: st.shard0.shardName}));
assert.commandWorked(st.s.adminCommand({shardCollection: fullCollName, key: {x: 1}}));
jsTestLog(st.s.getDB("config").collections.find().toArray());
jsTestLog(st.s.getDB("config").chunks.find().toArray());
assert.commandWorked(st.s.getDB(databaseName).runCommand({
createIndexes: collectionName,
indexes: [{key: {a: 1}, name: 'a_1'}],
}));
rewriteReplicaSetCatalog(st.rs0, fullCollName);
st.rs0.nodes.forEach((node) => {
assertExpectedCollectionIndexSpecWithNamespaceField(node, true);
});
// Verify shard1 has no collection.
assert.eq(
assert
.commandWorked(st.rs1.getPrimary().getDB(databaseName).runCommand({listCollections: 1}))
.cursor.firstBatch.length,
0);
assert.commandWorked(
st.s.adminCommand({moveChunk: fullCollName, find: {x: 0}, to: st.shard1.shardName}));
// Verify shard1 has cloned collection.
assert.eq(
assert
.commandWorked(st.rs1.getPrimary().getDB(databaseName).runCommand({listCollections: 1}))
.cursor.firstBatch.length,
1);
// Check old format metadata is not cloned. Even if setFCV happens concurrently with migrations,
// the deprecated metadata cannot be created elsewhere by cloning. Thus the setFCV step is
// guaranteed to completely remove the deprecated fields.
st.rs1.nodes.forEach((node) => {
assertExpectedCollectionIndexSpecWithNamespaceField(node, false);
});
// Double check old format metadata is still on shard0 (primary shard always owns the
// collection).
st.rs0.nodes.forEach((node) => {
assertExpectedCollectionIndexSpecWithNamespaceField(node, true);
});
// Implicitly verify that metadata consistency checks do not fail due to old metadata being in
// shard0 but not being in shard1.
st.stop();
}
testForReplicaSet();
testForShardedCluster();

28
src/mongo/db/catalog/coll_mod.cpp
View File

@ -803,6 +803,8 @@ Status _collModInternal(OperationContext* opCtx,
CollectionAcquisition* acquisition,
BSONObjBuilder* result,
boost::optional<repl::OplogApplication::Mode> mode) {
const auto fcvSnapshot = serverGlobalParams.featureCompatibility.acquireFCVSnapshot();
// Get key pattern from the config server if we may need it for parsing checks if on the primary
// before taking any locks. The ddl lock will prevent the key pattern from changing.
boost::optional<ShardKeyPattern> shardKeyPattern;
@ -1028,29 +1030,41 @@ Status _collModInternal(OperationContext* opCtx,
if (changed) {
writableColl->setTimeseriesOptions(opCtx, newOptions);
if (feature_flags::gTSBucketingParametersUnchanged.isEnabled(
VersionContext::getDecoration(opCtx),
serverGlobalParams.featureCompatibility.acquireFCVSnapshot())) {
VersionContext::getDecoration(opCtx), fcvSnapshot)) {
writableColl->setTimeseriesBucketingParametersChanged(opCtx, true);
};
}
}
const auto version = fcvSnapshot.getVersion();
// We involve an empty collMod command during a setFCV downgrade to clean timeseries
// bucketing parameters in the catalog. So if the FCV is in downgrading or downgraded stage,
// remove time-series bucketing parameters flag, as nodes older than 7.1 cannot understand
// this flag.
// (Generic FCV reference): This FCV check should exist across LTS binary versions.
// TODO SERVER-80003 remove special version handling when LTS becomes 8.0.
if (const auto version =
serverGlobalParams.featureCompatibility.acquireFCVSnapshot().getVersion();
cmrNew.numModifications == 0 && coll->timeseriesBucketingParametersHaveChanged() &&
if (cmrNew.numModifications == 0 && coll->timeseriesBucketingParametersHaveChanged() &&
version == multiversion::GenericFCV::kDowngradingFromLatestToLastLTS) {
writableColl->setTimeseriesBucketingParametersChanged(opCtx, boost::none);
}
// Fix any invalid index options for indexes belonging to this collection.
// Fix any invalid index options for indexes belonging to this collection, only for empty
// collMod requests which are called during setFCV upgrade.
const auto removeDeprecatedFields = [&]() {
if (cmrNew.numModifications > 0) {
return false;
}
if (!ServerGlobalParams::FCVSnapshot::isUpgradingOrDowngrading(version)) {
return false;
}
const auto transitionInfo = getTransitionFCVInfo(version);
return transitionInfo.from < transitionInfo.to;
}();
std::vector<std::string> indexesWithInvalidOptions =
writableColl->repairInvalidIndexOptions(opCtx);
writableColl->repairInvalidIndexOptions(opCtx, removeDeprecatedFields);
for (const auto& indexWithInvalidOptions : indexesWithInvalidOptions) {
const IndexDescriptor* desc =
writableColl->getIndexCatalog()->findIndexByName(opCtx, indexWithInvalidOptions);

7
src/mongo/db/catalog/collection.h
View File

@ -534,9 +534,12 @@ public:
/**
* Repairs invalid index options on all indexes in this collection. Returns a list of
* index names that were repaired.
* index names that were repaired. Specifying 'removeDeprecatedFields' as true, causes
* deprecated fields, which much be otherwise supported for backwards compatibility, to be
* removed when performing the repair.
*/
virtual std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) = 0;
virtual std::vector<std::string> repairInvalidIndexOptions(
OperationContext* opCtx, bool removeDeprecatedFields = false) = 0;
/**
* Updates the 'temp' setting for this collection.

13
src/mongo/db/catalog/collection_impl.cpp
View File

@ -1616,21 +1616,28 @@ void CollectionImpl::updatePrepareUniqueSetting(OperationContext* opCtx,
});
}
std::vector<std::string> CollectionImpl::repairInvalidIndexOptions(OperationContext* opCtx) {
std::vector<std::string> CollectionImpl::repairInvalidIndexOptions(OperationContext* opCtx,
bool removeDeprecatedFields) {
std::vector<std::string> indexesWithInvalidOptions;
const auto& allowedFieldNames = removeDeprecatedFields
? index_key_validate::kNonDeprecatedAllowedFieldNames
: index_key_validate::kAllowedFieldNames;
_writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) {
for (auto& index : md.indexes) {
if (index.isPresent()) {
BSONObj oldSpec = index.spec;
Status status = index_key_validate::validateIndexSpec(opCtx, oldSpec).getStatus();
Status status =
index_key_validate::validateIndexSpec(opCtx, oldSpec, allowedFieldNames)
.getStatus();
if (status.isOK()) {
continue;
}
indexesWithInvalidOptions.push_back(std::string(index.nameStringData()));
index.spec = index_key_validate::repairIndexSpec(md.nss, oldSpec);
index.spec =
index_key_validate::repairIndexSpec(md.nss, oldSpec, allowedFieldNames);
}
}
});

3
src/mongo/db/catalog/collection_impl.h
View File

@ -369,7 +369,8 @@ public:
StringData idxName,
bool prepareUnique) final;
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) final;
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx,
bool removeDeprecatedFields) final;
void setIsTemp(OperationContext* opCtx, bool isTemp) final;

3
src/mongo/db/catalog/collection_mock.h
View File

@ -417,7 +417,8 @@ public:
MONGO_UNREACHABLE;
}
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) override {
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx,
bool removeDeprecatedFields) override {
MONGO_UNREACHABLE;
}

3
src/mongo/db/catalog/index_catalog_impl.cpp
View File

@ -1651,9 +1651,6 @@ const IndexDescriptor* IndexCatalogImpl::refreshEntry(OperationContext* opCtx,
const std::string indexName = oldDesc->indexName();
invariant(collection->isIndexReady(indexName));
// The _id index should not be modified by a collMod.
tassert(9037800, "Should not be refreshing the _id index", !oldDesc->isIdIndex());
// Delete the IndexCatalogEntry that owns this descriptor. After deletion, 'oldDesc' is invalid
// and should not be dereferenced. Also, invalidate the index from the
// CollectionIndexUsageTrackerDecoration (shared state among Collection instances).

68
src/mongo/db/catalog/index_key_validate.cpp
View File

@ -88,6 +88,45 @@ std::function<void(std::map<StringData, std::set<IndexType>>&)> filterAllowedInd
using IndexVersion = IndexDescriptor::IndexVersion;
std::map<StringData, std::set<IndexType>> kAllowedFieldNames = {
{IndexDescriptor::k2dIndexBitsFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dIndexMaxFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dIndexMinFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dsphereCoarsestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
{IndexDescriptor::k2dsphereFinestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
{IndexDescriptor::k2dsphereVersionFieldName,
{IndexType::INDEX_2DSPHERE, IndexType::INDEX_2DSPHERE_BUCKET}},
{IndexDescriptor::kBackgroundFieldName, {}},
{IndexDescriptor::kCollationFieldName, {}},
{IndexDescriptor::kDefaultLanguageFieldName, {}},
{IndexDescriptor::kDropDuplicatesFieldName, {}},
{IndexDescriptor::kExpireAfterSecondsFieldName, {}},
{IndexDescriptor::kHiddenFieldName, {}},
{IndexDescriptor::kIndexNameFieldName, {}},
{IndexDescriptor::kIndexVersionFieldName, {}},
{IndexDescriptor::kKeyPatternFieldName, {}},
{IndexDescriptor::kLanguageOverrideFieldName, {}},
// TODO(SERVER-100328): remove after 9.0 is branched.
{IndexDescriptor::kNamespaceFieldName, {}},
{IndexDescriptor::kPartialFilterExprFieldName, {}},
{IndexDescriptor::kWildcardProjectionFieldName, {IndexType::INDEX_WILDCARD}},
{IndexDescriptor::kSparseFieldName, {}},
{IndexDescriptor::kStorageEngineFieldName, {}},
{IndexDescriptor::kTextVersionFieldName, {IndexType::INDEX_TEXT}},
{IndexDescriptor::kUniqueFieldName, {}},
{IndexDescriptor::kWeightsFieldName, {IndexType::INDEX_TEXT}},
{IndexDescriptor::kOriginalSpecFieldName, {}},
{IndexDescriptor::kPrepareUniqueFieldName, {}},
// Index creation under legacy writeMode can result in an index spec with an _id field.
{"_id", {}},
// TODO SERVER-76108: Field names are not validated to match index type. This was used for the
// removed 'geoHaystack' index type, but users could have set it for other index types as well.
// We need to keep allowing it until FCV upgrade is implemented to clean this up.
{"bucketSize"_sd, {}}};
// Initialised by a GlobalInitializerRegisterer.
std::map<StringData, std::set<IndexType>> kNonDeprecatedAllowedFieldNames = {};
namespace {
// When the skipIndexCreateFieldNameValidation failpoint is enabled, validation for index field
// names will be disabled. This will allow for creation of indexes with invalid field names in their
@ -103,6 +142,7 @@ static const std::set<StringData> allowedIdIndexFieldNames = {
IndexDescriptor::kIndexNameFieldName,
IndexDescriptor::kIndexVersionFieldName,
IndexDescriptor::kKeyPatternFieldName,
// TODO(SERVER-100328): remove after 9.0 is branched.
IndexDescriptor::kNamespaceFieldName,
// Index creation under legacy writeMode can result in an index spec with an _id field.
"_id"};
@ -285,7 +325,7 @@ BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec)
auto appendIndexSpecFn = [](const BSONElement& indexSpecElem, BSONObjBuilder* builder) {
builder->append(indexSpecElem);
};
return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, appendIndexSpecFn);
return buildRepairedIndexSpec(ns, indexSpec, kAllowedFieldNames, appendIndexSpecFn);
}
BSONObj repairIndexSpec(const NamespaceString& ns,
@ -344,7 +384,10 @@ BSONObj repairIndexSpec(const NamespaceString& ns,
return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, fixIndexSpecFn);
}
StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& indexSpec) {
StatusWith<BSONObj> validateIndexSpec(
OperationContext* opCtx,
const BSONObj& indexSpec,
const std::map<StringData, std::set<IndexType>>& allowedFieldNames) {
bool hasKeyPatternField = false;
bool hasIndexNameField = false;
bool hasNamespaceField = false;
@ -359,7 +402,7 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
auto clusteredField = indexSpec[IndexDescriptor::kClusteredFieldName];
bool apiStrict = opCtx && APIParameters::get(opCtx).getAPIStrict().value_or(false);
auto fieldNamesValidStatus = validateIndexSpecFieldNames(indexSpec);
auto fieldNamesValidStatus = validateIndexSpecFieldNames(indexSpec, allowedFieldNames);
if (!fieldNamesValidStatus.isOK()) {
return fieldNamesValidStatus;
}
@ -438,6 +481,7 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
}
} else if (IndexDescriptor::kNamespaceFieldName == indexSpecElemFieldName) {
// TODO(SERVER-100328): remove after 9.0 is branched.
hasNamespaceField = true;
} else if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) {
if (!indexSpecElem.isNumber()) {
@ -686,6 +730,7 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
// Ignore any 'ns' field in the index spec because this field is dropped post-4.0. Don't remove
// the field during repair, as repair may run on old data files (version 3.6 and 4.0) that
// require the field to be present.
// TODO(SERVER-100328): remove after 9.0 is branched.
if (hasNamespaceField && !storageGlobalParams.repair) {
modifiedSpec = modifiedSpec.removeField(IndexDescriptor::kNamespaceFieldName);
}
@ -794,7 +839,8 @@ Status validateClusteredSpecFieldNames(const BSONObj& indexSpec) {
* value, is the the sub-module's responsibility to ensure that the content is valid and that only
* expected fields are present at creation time
*/
Status validateIndexSpecFieldNames(const BSONObj& indexSpec) {
Status validateIndexSpecFieldNames(
const BSONObj& indexSpec, const std::map<StringData, std::set<IndexType>>& allowedFieldNames) {
if (MONGO_unlikely(skipIndexCreateFieldNameValidation.shouldFail())) {
return Status::OK();
}
@ -1082,8 +1128,20 @@ BSONObj parseAndValidateIndexSpecs(OperationContext* opCtx, const BSONObj& index
GlobalInitializerRegisterer filterAllowedIndexFieldNamesInitializer(
"FilterAllowedIndexFieldNames", [](InitializerContext*) {
if (filterAllowedIndexFieldNames)
filterAllowedIndexFieldNames(allowedFieldNames);
filterAllowedIndexFieldNames(kAllowedFieldNames);
});
GlobalInitializerRegisterer nonDeprecatedAllowedFieldNamesInitializer(
"NonDeprecatedAllowedIndexFieldNames",
[](InitializerContext*) {
kNonDeprecatedAllowedFieldNames = kAllowedFieldNames;
for (const auto& name : kDeprecatedFieldNames) {
kNonDeprecatedAllowedFieldNames.erase(name);
}
},
nullptr,
{"FilterAllowedIndexFieldNames"});
} // namespace index_key_validate
} // namespace mongo

61
src/mongo/db/catalog/index_key_validate.h
View File

@ -67,41 +67,22 @@ constexpr auto kExpireAfterSecondsForInactiveTTLIndex =
* Describe which field names are considered valid options when creating an index. If the set
* associated with the field name is empty, the option is always valid, otherwise it will be allowed
* only when creating the set of index types listed in the set.
*
* Although the variable is not defined as const, it in practice is one. It may be modified only by
* a GlobalInitializerRegisterer.
*/
static std::map<StringData, std::set<IndexType>> allowedFieldNames = {
{IndexDescriptor::k2dIndexBitsFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dIndexMaxFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dIndexMinFieldName, {IndexType::INDEX_2D}},
{IndexDescriptor::k2dsphereCoarsestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
{IndexDescriptor::k2dsphereFinestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
{IndexDescriptor::k2dsphereVersionFieldName,
{IndexType::INDEX_2DSPHERE, IndexType::INDEX_2DSPHERE_BUCKET}},
{IndexDescriptor::kBackgroundFieldName, {}},
{IndexDescriptor::kCollationFieldName, {}},
{IndexDescriptor::kDefaultLanguageFieldName, {}},
{IndexDescriptor::kDropDuplicatesFieldName, {}},
{IndexDescriptor::kExpireAfterSecondsFieldName, {}},
{IndexDescriptor::kHiddenFieldName, {}},
{IndexDescriptor::kIndexNameFieldName, {}},
{IndexDescriptor::kIndexVersionFieldName, {}},
{IndexDescriptor::kKeyPatternFieldName, {}},
{IndexDescriptor::kLanguageOverrideFieldName, {}},
{IndexDescriptor::kNamespaceFieldName, {}},
{IndexDescriptor::kPartialFilterExprFieldName, {}},
{IndexDescriptor::kWildcardProjectionFieldName, {IndexType::INDEX_WILDCARD}},
{IndexDescriptor::kSparseFieldName, {}},
{IndexDescriptor::kStorageEngineFieldName, {}},
{IndexDescriptor::kTextVersionFieldName, {IndexType::INDEX_TEXT}},
{IndexDescriptor::kUniqueFieldName, {}},
{IndexDescriptor::kWeightsFieldName, {IndexType::INDEX_TEXT}},
{IndexDescriptor::kOriginalSpecFieldName, {}},
{IndexDescriptor::kPrepareUniqueFieldName, {}},
// Index creation under legacy writeMode can result in an index spec with an _id field.
{"_id", {}},
// TODO SERVER-76108: Field names are not validated to match index type. This was used for the
// removed 'geoHaystack' index type, but users could have set it for other index types as well.
// We need to keep allowing it until FCV upgrade is implemented to clean this up.
{"bucketSize"_sd, {}}};
extern std::map<StringData, std::set<IndexType>> kAllowedFieldNames;
const std::set<StringData> kDeprecatedFieldNames = {
"_id"_sd, "bucketSize"_sd, IndexDescriptor::kNamespaceFieldName};
/**
* Like 'allowedFieldNames', but removes deprecated fields specified in kDeprecatedFieldNames.
*
* Although the variable is not defined as const, it in practice is one. It may be modified only by
* a GlobalInitializerRegisterer.
*/
extern std::map<StringData, std::set<IndexType>> kNonDeprecatedAllowedFieldNames;
/**
* Checks if the key is valid for building an index according to the validation rules for the given
@ -114,7 +95,11 @@ Status validateKeyPattern(const BSONObj& key, IndexDescriptor::IndexVersion inde
* has any missing attributes filled in. If the index specification is malformed, then an error
* status is returned.
*/
StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& indexSpec);
StatusWith<BSONObj> validateIndexSpec(
OperationContext* opCtx,
const BSONObj& indexSpec,
const std::map<StringData, std::set<IndexType>>& allowedFieldNames =
index_key_validate::kAllowedFieldNames);
/**
* Returns a new index spec with any unknown field names removed from 'indexSpec'.
@ -128,7 +113,7 @@ BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec)
BSONObj repairIndexSpec(const NamespaceString& ns,
const BSONObj& indexSpec,
const std::map<StringData, std::set<IndexType>>& allowedFieldNames =
index_key_validate::allowedFieldNames);
index_key_validate::kAllowedFieldNames);
/**
* Performs additional validation for _id index specifications. This should be called after
@ -140,7 +125,9 @@ Status validateIdIndexSpec(const BSONObj& indexSpec);
* Confirms that 'indexSpec' contains only valid field names. Returns an error if an unexpected
* field name is found.
*/
Status validateIndexSpecFieldNames(const BSONObj& indexSpec);
Status validateIndexSpecFieldNames(const BSONObj& indexSpec,
const std::map<StringData, std::set<IndexType>>&
allowedFieldNames = index_key_validate::kAllowedFieldNames);
/**
* Validates the 'collation' field in the index specification 'indexSpec' and fills in the full

3
src/mongo/db/catalog/virtual_collection_impl.h
View File

@ -368,7 +368,8 @@ public:
unimplementedTasserted();
}
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) final {
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx,
bool removeDeprecatedFields) final {
unimplementedTasserted();
return {};
}

32
src/mongo/db/commands/set_feature_compatibility_version_command.cpp
View File

@ -782,6 +782,8 @@ private:
_setShardedClusterCardinalityParameter(opCtx, requestedVersion);
_initializePlacementHistory(opCtx, requestedVersion);
}
_cleanUpDeprecatedCatalogMetadata(opCtx);
}
// TODO (SERVER-83704): Remove this function once 8.0 becomes last LTS.
@ -796,6 +798,36 @@ private:
}
}
// TODO(SERVER-100328): remove after 9.0 is branched.
// WARNING: do not rely on this method to clean up metadata that can be created concurrently. It
// is fine to rely on this only when missing concurrently created collections is fine, when
// newly created collections no longer use the metadata format we wish to remove.
void _cleanUpDeprecatedCatalogMetadata(OperationContext* opCtx) {
// We bypass the UserWritesBlock mode here in order to not see errors arising from the
// block. The user already has permission to run FCV at this point and the writes performed
// here aren't modifying any user data with the exception of fixing up the collection
// metadata.
auto originalValue = WriteBlockBypass::get(opCtx).isWriteBlockBypassEnabled();
ON_BLOCK_EXIT([&] { WriteBlockBypass::get(opCtx).set(originalValue); });
WriteBlockBypass::get(opCtx).set(true);
for (const auto& dbName : DatabaseHolder::get(opCtx)->getNames()) {
Lock::DBLock dbLock(opCtx, dbName, MODE_IX);
catalog::forEachCollectionFromDb(
opCtx, dbName, MODE_X, [&](const Collection* collection) {
// To remove deprecated catalog metadata, issue a collmod with no other options
// set.
BSONObjBuilder responseBuilder;
uassertStatusOK(processCollModCommand(opCtx,
collection->ns(),
CollMod{collection->ns()},
nullptr,
&responseBuilder));
return true;
});
}
}
void _dropSessionsCollectionLocally(OperationContext* opCtx,
const FCV requestedVersion,
const FCV originalVersion) {

4
src/mongo/db/index/index_descriptor.cpp
View File

@ -93,7 +93,8 @@ std::map<StringData, BSONElement> populateOptionsMapForEqualityCheck(const BSONO
// These index options are not considered for equality.
static const StringDataSet kIndexOptionsNotConsideredForEqualityCheck{
IndexDescriptor::kKeyPatternFieldName, // checked specially
IndexDescriptor::kKeyPatternFieldName, // checked specially
// TODO(SERVER-100328): remove after 9.0 is branched.
IndexDescriptor::kNamespaceFieldName, // removed in 4.4
IndexDescriptor::kIndexNameFieldName, // checked separately
IndexDescriptor::kIndexVersionFieldName, // not considered for equivalence
@ -137,6 +138,7 @@ constexpr StringData IndexDescriptor::kIndexNameFieldName;
constexpr StringData IndexDescriptor::kIndexVersionFieldName;
constexpr StringData IndexDescriptor::kKeyPatternFieldName;
constexpr StringData IndexDescriptor::kLanguageOverrideFieldName;
// TODO(SERVER-100328): remove after 9.0 is branched.
constexpr StringData IndexDescriptor::kNamespaceFieldName;
constexpr StringData IndexDescriptor::kPartialFilterExprFieldName;
constexpr StringData IndexDescriptor::kWildcardProjectionFieldName;

1
src/mongo/db/index/index_descriptor.h
View File

@ -94,6 +94,7 @@ public:
static constexpr StringData kIndexVersionFieldName = "v"_sd;
static constexpr StringData kKeyPatternFieldName = "key"_sd;
static constexpr StringData kLanguageOverrideFieldName = "language_override"_sd;
// TODO(SERVER-100328): remove after 9.0 is branched.
static constexpr StringData kNamespaceFieldName = "ns"_sd; // Removed in 4.4
static constexpr StringData kPartialFilterExprFieldName = "partialFilterExpression"_sd;
static constexpr StringData kWildcardProjectionFieldName = "wildcardProjection"_sd;

3
src/mongo/db/query/timeseries/timeseries_rewrites_mocks.h
View File

@ -392,7 +392,8 @@ public:
MONGO_UNREACHABLE;
}
std::vector<std::string> repairInvalidIndexOptions(OperationContext*) override {
std::vector<std::string> repairInvalidIndexOptions(OperationContext*,
bool removeDeprecatedFields) override {
MONGO_UNREACHABLE;
}

3
src/mongo/db/s/migration_chunk_cloner_source_test.cpp
View File

@ -395,7 +395,8 @@ public:
MONGO_UNREACHABLE;
}
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) override {
std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx,
bool removeDeprecatedFields) override {
MONGO_UNREACHABLE;
}

7
src/mongo/db/s/shard_server_op_observer.cpp
View File

@ -941,7 +941,12 @@ void ShardServerOpObserver::onCollMod(OperationContext* opCtx,
if (repl::ReplicationCoordinator::get(opCtx)->isInInitialSyncOrRollback()) {
return;
}
abortOngoingMigrationIfNeeded(opCtx, nss);
// A collMod with no arguments is used to repair or cleanup metadata during FCV upgrade or
// downgrade. This is not an index modification operation, and does not need to abort ongoing
// migrations.
if (collModCmd.nFields() > 1) {
abortOngoingMigrationIfNeeded(opCtx, nss);
}
};
void ShardServerOpObserver::onReplicationRollback(OperationContext* opCtx,

18
src/mongo/db/s/sharding_recovery_service.cpp
View File

@ -285,6 +285,12 @@ void ShardingRecoveryService::acquireRecoverableCriticalSectionBlockWrites(
return;
}
// TODO(SERVER-100328): remove after 9.0 is branched.
// If the storage snapshot is reused between the 'find' above and the 'insert' below, the
// operation may unnecessarily fail if a collMod is run on the
// kCollectionCriticalSectionsNamespace during FCV upgrade.
shard_role_details::getRecoveryUnit(opCtx)->abandonSnapshot();
// The collection critical section is not taken, try to acquire it.
// The following code will try to add a doc to config.criticalCollectionSections:
@ -400,6 +406,12 @@ void ShardingRecoveryService::promoteRecoverableCriticalSectionToBlockAlsoReads(
return;
}
// TODO(SERVER-100328): remove after 9.0 is branched.
// If the storage snapshot is reused between the 'find' above and the 'update' below, the
// operation may unnecessarily fail when a collMod is run on the
// kCollectionCriticalSectionsNamespace during FCV upgrade.
shard_role_details::getRecoveryUnit(opCtx)->abandonSnapshot();
// The CS is in the catch-up phase, try to advance it to the commit phase.
// The following code will try to update a doc from config.criticalCollectionSections:
@ -534,6 +546,12 @@ void ShardingRecoveryService::releaseRecoverableCriticalSection(
beforeReleasingAction(opCtx, nss);
// TODO(SERVER-100328): remove after 9.0 is branched.
// If the storage snapshot is reused between the 'find' above and the 'delete' below, the
// operation may unnecessarily fail if a collMod is run on the
// kCollectionCriticalSectionsNamespace during FCV upgrade.
shard_role_details::getRecoveryUnit(opCtx)->abandonSnapshot();
// The following code will try to remove a doc from config.criticalCollectionSections:
// - If everything goes well, the shard server op observer will release the in-memory CS
// - Otherwise this call will fail and the CS won't be released (neither persisted nor

21
src/mongo/shell/shell_utils_extended.cpp
View File

@ -540,7 +540,25 @@ BSONObj decompressBSONColumn(const BSONObj& a, void* data) {
* Dumps BSON data as a Hex-formatted string
*/
BSONObj dumpBSONAsHex(const BSONObj& a, void* data) {
return BSON("" << bson_bin_util::toHex(a));
uassert(9174601,
"dumpBSONAsHex() takes one argument: a BSON obj",
a.nFields() == 1 && a.firstElementType() == Object);
auto obj = a.firstElement().Obj();
return BSON("" << hexblob::encodeLower(obj.objdata(), obj.objsize()));
}
/**
* Convert a hex string encoded BSON into its BSONObj form.
*/
BSONObj hexToBSON(const BSONObj& a, void*) {
uassert(9174600,
"hexToBSON takes one argument: a hex string",
a.nFields() == 1 && a.firstElementType() == String);
BufBuilder bb;
hexblob::decode(a.firstElement().String(), &bb);
return BSON("" << BSONObj(bb.release()));
}
BSONObj generateStorageBSON(const BSONObj& args, void* data) {
@ -881,6 +899,7 @@ void installShellUtilsExtended(Scope& scope) {
scope.injectNative("getFileMode", getFileMode);
scope.injectNative("decompressBSONColumn", decompressBSONColumn);
scope.injectNative("dumpBSONAsHex", dumpBSONAsHex);
scope.injectNative("hexToBSON", hexToBSON);
scope.injectNative("_copyFileRange", copyFileRange);
scope.injectNative("_readDumpFile", readDumpFile);
scope.injectNative("_numObjsInDumpFile", numObjsInDumpFile);