mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-107873 Report number of Queryable Encryption collections using each index type in serverStatus (#42303)

Browse Source 
GitOrigin-RevId: e14ecda06812a4fdb7d490484c22c0ab952f51bf
This commit is contained in:
Erwin Pe 2025-10-08 16:05:26 -04:00 committed by MongoDB Bot
parent e32ffb05e4
commit a8b0cb11c0
8 changed files with 322 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/mongo/crypto/encryption_fields_util.cpp
View File

@ -50,4 +50,36 @@ boost::optional<EncryptedFieldMatchResult> findMatchingEncryptedField(
return {{*itr, key.numParts() <= itr->numParts()}};
}
bool visitQueryTypeConfigs(const EncryptedField& field,
const QueryTypeConfigVisitor& visitOne,
const UnindexedEncryptedFieldVisitor& onEmptyField) {
if (!field.getQueries()) {
if (onEmptyField) {
return onEmptyField(field);
}
return false;
}
return visit(OverloadedVisitor{[&](QueryTypeConfig query) { return visitOne(field, query); },
[&](std::vector<QueryTypeConfig> queries) {
return std::any_of(queries.cbegin(),
queries.cend(),
[&](const QueryTypeConfig& qtc) {
return visitOne(field, qtc);
});
}},
field.getQueries().get());
}
bool visitQueryTypeConfigs(const EncryptedFieldConfig& efc,
const QueryTypeConfigVisitor& visitOne,
const UnindexedEncryptedFieldVisitor& onEmptyField) {
for (const auto& field : efc.getFields()) {
if (visitQueryTypeConfigs(field, visitOne, onEmptyField)) {
return true;
}
}
return false;
}
} // namespace mongo

35
src/mongo/crypto/encryption_fields_util.h
View File

@ -212,4 +212,39 @@ struct EncryptedFieldMatchResult {
boost::optional<EncryptedFieldMatchResult> findMatchingEncryptedField(
const FieldRef& key, const std::vector<FieldRef>& encryptedFields);
/**
* Function to evaluate a QueryTypeConfig that appears under an EncryptedField. Used as a
* visitor function when iterating all QueryTypeConfig present in an EncryptedFieldConfig.
* A return value of true signals that a condition has been met and iteration may halt.
*/
using QueryTypeConfigVisitor = std::function<bool(const EncryptedField&, const QueryTypeConfig&)>;
/**
* Function to evaluate an EncryptedField that has no QueryTypeConfig (i.e. unindexed). Used as a
* visitor function when iterating all EncryptedField present in an EncryptedFieldConfig.
* A return value of true signals that a condition has been met and iteration may halt.
*/
using UnindexedEncryptedFieldVisitor = std::function<bool(const EncryptedField&)>;
/**
* For each QueryTypeConfig present under the EncryptedField, invokes the visitor function visit.
* If the EncryptedField does not have any QueryTypeConfig, invokes the visitor function
* onEmptyField. Immediately returns true once visit or onEmptyField has returned true. Returns
* false if the visitor functions returned false on all QueryTypeConfig.
*/
bool visitQueryTypeConfigs(const EncryptedField& field,
const QueryTypeConfigVisitor& visit,
const UnindexedEncryptedFieldVisitor& onEmptyField = nullptr);
/**
* For each QueryTypeConfig present under each EncryptedField in the EncryptedFieldConfig, invokes
* the visitor function visit. If an EncryptedField does not have any QueryTypeConfig, invokes the
* visitor function onEmptyField. Immediately returns true once visit or onEmptyField has returned
* true. Returns false if the visitor functions returned false on all QueryTypeConfig.
*/
bool visitQueryTypeConfigs(const EncryptedFieldConfig& efc,
const QueryTypeConfigVisitor& visit,
const UnindexedEncryptedFieldVisitor& onEmptyField = nullptr);
} // namespace mongo

51
src/mongo/crypto/fle_crypto.cpp
View File

@ -3978,13 +3978,14 @@ void EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceed
return;
}
auto checkOneQueryType = [](StringData path, const QueryTypeConfig& qtc) {
auto checkOneQueryType = [](const EncryptedField& field, const QueryTypeConfig& qtc) {
if (qtc.getQueryType() != QueryTypeEnum::SubstringPreview) {
return;
return false;
}
int32_t ub = qtc.getStrMaxQueryLength().get();
int32_t lb = qtc.getStrMinQueryLength().get();
int32_t max = qtc.getStrMaxLength().get();
auto path = field.getPath();
uassert(10453200,
fmt::format("strMinQueryLength ({}) must be >= {} for substringPreview query "
"type of field {}. {}",
@ -4009,22 +4010,10 @@ void EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceed
path,
bypassMsg),
max <= kSubstringPreviewMaxLengthMax);
return false;
};
for (const auto& field : ef.getFields()) {
if (!field.getQueries()) {
continue;
}
visit(
OverloadedVisitor{[&](QueryTypeConfig qtc) { checkOneQueryType(field.getPath(), qtc); },
[&](std::vector<QueryTypeConfig> queries) {
for (auto& qtc : queries) {
checkOneQueryType(field.getPath(), qtc);
}
}},
field.getQueries().get());
}
visitQueryTypeConfigs(ef, checkOneQueryType);
}
std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(ConstDataRange cdr) {
@ -4207,29 +4196,17 @@ ConstDataRange binDataToCDR(BSONElement element) {
}
bool hasQueryTypeMatching(const EncryptedField& field, const QueryTypeMatchFn& matcher) {
if (!field.getQueries()) {
return false;
}
return visit(OverloadedVisitor{
[&](QueryTypeConfig query) { return matcher(query.getQueryType()); },
[&](std::vector<QueryTypeConfig> queries) {
return std::any_of(
queries.cbegin(), queries.cend(), [&](const QueryTypeConfig& qtc) {
return matcher(qtc.getQueryType());
});
}},
field.getQueries().get());
return visitQueryTypeConfigs(field,
[&matcher](const EncryptedField&, const QueryTypeConfig& qtc) {
return matcher(qtc.getQueryType());
});
}
bool hasQueryTypeMatching(const EncryptedFieldConfig& config, const QueryTypeMatchFn& matcher) {
for (const auto& field : config.getFields()) {
if (field.getQueries().has_value()) {
bool hasQuery = hasQueryTypeMatching(field, matcher);
if (hasQuery) {
return hasQuery;
}
}
}
return false;
return visitQueryTypeConfigs(config,
[&matcher](const EncryptedField&, const QueryTypeConfig& qtc) {
return matcher(qtc.getQueryType());
});
}
bool hasQueryType(const EncryptedField& field, QueryTypeEnum queryType) {

69
src/mongo/crypto/fle_stats.cpp
View File

@ -29,21 +29,18 @@
#include "mongo/crypto/fle_stats.h"
#include "mongo/bson/bsonmisc.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/crypto/encryption_fields_util.h"
#include "mongo/crypto/fle_options_gen.h"
#include "mongo/util/system_tick_source.h"
#include "mongo/util/testing_options_gen.h"
#include <memory>
namespace mongo {
namespace {
// We only track fle stats on the shard.
auto& fleStatusSection =
*ServerStatusSectionBuilder<FLEStatusSection>("fle").bind(globalSystemTickSource()).forShard();
*ServerStatusSectionBuilder<FLEStatusSection>("fle").forShard().bind(globalSystemTickSource());
} // namespace
FLEStatusSection::FLEStatusSection(std::string name, ClusterRole role, TickSource* tickSource)
@ -91,6 +88,16 @@ BSONObj FLEStatusSection::generateSection(OperationContext* opCtx,
sub << "totalMillis" << emuBinaryTotalMillis.loadRelaxed();
}
{
FLEIndexTypeStats temp;
{
stdx::lock_guard<stdx::mutex> lock(_indexTypeMutex);
temp = _indexTypeStats;
}
auto sub = BSONObjBuilder(builder.subobjStart("indexTypeStats"));
temp.serialize(&sub);
}
return builder.obj();
}
@ -99,4 +106,56 @@ FLEStatusSection::EmuBinaryTracker FLEStatusSection::makeEmuBinaryTracker() {
return EmuBinaryTracker(this, gTestingDiagnosticsEnabledAtStartup);
}
void FLEStatusSection::updateIndexTypeStats(const EncryptedFieldConfig& efc, bool subtract) {
const std::int64_t delta = (subtract ? -1 : 1);
FLEIndexTypeStats deltas;
visitQueryTypeConfigs(
efc,
[&deltas, delta](const EncryptedField& field, const QueryTypeConfig& qtc) {
switch (qtc.getQueryType()) {
case QueryTypeEnum::Equality:
deltas.setEquality(delta);
break;
case QueryTypeEnum::Range:
deltas.setRange(delta);
break;
case QueryTypeEnum::RangePreviewDeprecated:
deltas.setRangePreview(delta);
break;
case QueryTypeEnum::SubstringPreview:
deltas.setSubstringPreview(delta);
break;
case QueryTypeEnum::SuffixPreview:
deltas.setSuffixPreview(delta);
break;
case QueryTypeEnum::PrefixPreview:
deltas.setPrefixPreview(delta);
break;
default:
MONGO_UNREACHABLE;
};
return false;
},
[&deltas, delta](const EncryptedField&) {
deltas.setUnindexed(delta);
return false;
});
stdx::lock_guard<stdx::mutex> lock(_indexTypeMutex);
FLEStatsUtil::accumulateStats(_indexTypeStats, deltas);
}
void FLEStatusSection::updateIndexTypeStatsOnRegisterCollection(const EncryptedFieldConfig& efc) {
updateIndexTypeStats(efc, false);
}
void FLEStatusSection::updateIndexTypeStatsOnDeregisterCollection(const EncryptedFieldConfig& efc) {
updateIndexTypeStats(efc, true);
}
void FLEStatusSection::clearIndexTypeStats() {
stdx::lock_guard<stdx::mutex> lock(_indexTypeMutex);
_indexTypeStats = {};
}
} // namespace mongo

34
src/mongo/crypto/fle_stats.h
View File

@ -40,9 +40,6 @@
#include "mongo/util/tick_source.h"
#include "mongo/util/timer.h"
#include <mutex>
namespace mongo {
namespace FLEStatsUtil {
@ -56,6 +53,15 @@ static void accumulateStats(ECOCStats& left, const ECOCStats& right) {
left.setRead(left.getRead() + right.getRead());
left.setDeleted(left.getDeleted() + right.getDeleted());
}
static void accumulateStats(FLEIndexTypeStats& left, const FLEIndexTypeStats& right) {
left.setEquality(left.getEquality() + right.getEquality());
left.setRange(left.getRange() + right.getRange());
left.setRangePreview(left.getRangePreview() + right.getRangePreview());
left.setSubstringPreview(left.getSubstringPreview() + right.getSubstringPreview());
left.setSuffixPreview(left.getSuffixPreview() + right.getSuffixPreview());
left.setPrefixPreview(left.getPrefixPreview() + right.getPrefixPreview());
left.setUnindexed(left.getUnindexed() + right.getUnindexed());
}
} // namespace FLEStatsUtil
/**
@ -69,9 +75,9 @@ public:
// Return the global status section Singleton
static FLEStatusSection& get();
// Report FLE metrics if any stat has been set.
// Report FLE metrics at all times
bool includeByDefault() const final {
return _hasStats.loadRelaxed();
return true;
}
BSONObj generateSection(OperationContext* opCtx, const BSONElement& configElement) const final;
@ -102,7 +108,6 @@ public:
explicit EmuBinaryTracker(FLEStatusSection* section, bool active)
: _section(section), _active(active), _timer(_section->_tickSource) {
if (_active) {
_section->_hasStats.store(true);
_section->emuBinaryCalls.fetchAndAddRelaxed(1);
}
}
@ -116,22 +121,24 @@ public:
void updateCompactionStats(const CompactStats& stats) {
stdx::lock_guard<stdx::mutex> lock(_compactMutex);
_hasStats.store(true);
FLEStatsUtil::accumulateStats(_compactStats.getEsc(), stats.getEsc());
FLEStatsUtil::accumulateStats(_compactStats.getEcoc(), stats.getEcoc());
}
void updateCleanupStats(const CleanupStats& stats) {
stdx::lock_guard<stdx::mutex> lock(_cleanupMutex);
_hasStats.store(true);
FLEStatsUtil::accumulateStats(_cleanupStats.getEsc(), stats.getEsc());
FLEStatsUtil::accumulateStats(_cleanupStats.getEcoc(), stats.getEcoc());
}
private:
TickSource* _tickSource;
void updateIndexTypeStatsOnRegisterCollection(const EncryptedFieldConfig& efc);
void updateIndexTypeStatsOnDeregisterCollection(const EncryptedFieldConfig& efc);
void clearIndexTypeStats();
AtomicWord<bool> _hasStats{false};
private:
void updateIndexTypeStats(const EncryptedFieldConfig& efc, bool subtract);
TickSource* _tickSource;
AtomicWord<long long> emuBinaryCalls;
AtomicWord<long long> emuBinarySuboperation;
@ -142,6 +149,11 @@ private:
mutable stdx::mutex _cleanupMutex;
CleanupStats _cleanupStats;
// Tracks and reports statistics about how many collections in the catalog use each of the
// Queryable Encryption index types, and how many collections use unindexed encryption.
mutable stdx::mutex _indexTypeMutex;
FLEIndexTypeStats _indexTypeStats;
};
} // namespace mongo

25
src/mongo/crypto/fle_stats.idl
View File

@ -70,3 +70,28 @@ structs:
fields:
ecoc: ECOCStats
esc: ECStats
FLEIndexTypeStats:
description: "Counters for each FLE2 index type, including unindexed"
fields:
unindexed:
type: exactInt64
default: 0
equality:
type: exactInt64
default: 0
range:
type: exactInt64
default: 0
substringPreview:
type: exactInt64
default: 0
suffixPreview:
type: exactInt64
default: 0
prefixPreview:
type: exactInt64
default: 0
rangePreview:
type: exactInt64
default: 0

126
src/mongo/crypto/fle_stats_test.cpp
View File

@ -30,8 +30,8 @@
#include "mongo/crypto/fle_stats.h"
#include "mongo/base/string_data.h"
#include "mongo/bson/bsonmisc.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/bson/json.h"
#include "mongo/db/service_context.h"
#include "mongo/db/service_context_test_fixture.h"
#include "mongo/idl/idl_parser.h"
@ -93,7 +93,7 @@ public:
};
TEST_F(FLEStatsTest, NoopStats) {
ASSERT_FALSE(instance->includeByDefault());
ASSERT_TRUE(instance->includeByDefault());
auto obj = instance->generateSection(opCtx, BSONElement());
ASSERT_TRUE(obj.hasField("compactStats"));
@ -136,7 +136,7 @@ TEST_F(FLEStatsTest, BinaryEmuStatsAreEmptyWithoutTesting) {
tracker.recordSuboperation();
}
ASSERT_FALSE(instance->includeByDefault());
ASSERT_TRUE(instance->includeByDefault());
auto obj = instance->generateSection(opCtx, BSONElement());
ASSERT_TRUE(obj.hasField("compactStats"));
@ -170,5 +170,125 @@ TEST_F(FLEStatsTest, BinaryEmuStatsArePopulatedWithTesting) {
ASSERT_EQ(100, obj["emuBinaryStats"]["totalMillis"].Long());
}
TEST_F(FLEStatsTest, IndexTypeStats) {
struct IndexTypeCounters {
int64_t equality = 0;
int64_t unindexed = 0;
int64_t range = 0;
int64_t rangePreview = 0;
int64_t substringPreview = 0;
int64_t suffixPreview = 0;
int64_t prefixPreview = 0;
};
auto assertCounters = [this](const IndexTypeCounters& expected) {
auto obj = instance->generateSection(opCtx, BSONElement());
auto actual =
FLEIndexTypeStats::parse(IDLParserContext("fle_stats"), obj["indexTypeStats"].Obj());
ASSERT_EQ(actual.getEquality(), expected.equality);
ASSERT_EQ(actual.getUnindexed(), expected.unindexed);
ASSERT_EQ(actual.getRange(), expected.range);
ASSERT_EQ(actual.getRangePreview(), expected.rangePreview);
ASSERT_EQ(actual.getSubstringPreview(), expected.substringPreview);
ASSERT_EQ(actual.getSuffixPreview(), expected.suffixPreview);
ASSERT_EQ(actual.getPrefixPreview(), expected.prefixPreview);
};
const auto buildConfig = [](const StringMap<int64_t>& spec) {
EncryptedFieldConfig efc;
std::vector<EncryptedField> fields;
const auto keyId = UUID::gen();
for (auto& [indexType, count] : spec) {
if (indexType == "unindexed") {
for (auto i = count; i > 0; i--) {
fields.emplace_back(keyId, indexType);
}
} else if (indexType == "multi") {
for (auto i = count; i > 0; i--) {
fields.emplace_back(keyId, indexType);
std::vector<QueryTypeConfig> queries;
queries.push_back(QueryTypeConfig::parse(
IDLParserContext("qtc"), fromjson(R"({"queryType": "suffixPreview"})")));
queries.push_back(QueryTypeConfig::parse(
IDLParserContext("qtc"), fromjson(R"({"queryType": "prefixPreview"})")));
fields.back().setQueries(
std::variant<std::vector<QueryTypeConfig>, QueryTypeConfig>(
std::move(queries)));
}
} else {
for (auto i = count; i > 0; i--) {
fields.emplace_back(keyId, indexType);
auto qtc =
QueryTypeConfig::parse(IDLParserContext("qtc"),
fromjson("{\"queryType\": \"" + indexType + "\"}"));
fields.back().setQueries(
std::variant<std::vector<QueryTypeConfig>, QueryTypeConfig>(
std::move(qtc)));
}
}
}
efc.setFields(std::move(fields));
return efc;
};
ASSERT_TRUE(instance->includeByDefault());
IndexTypeCounters expected;
assertCounters(expected);
auto efc1 = buildConfig({{"unindexed", 4}, {"equality", 2}, {"multi", 3}});
instance->updateIndexTypeStatsOnRegisterCollection(efc1);
expected.unindexed++;
expected.equality++;
expected.suffixPreview++;
expected.prefixPreview++;
assertCounters(expected);
auto efc2 = buildConfig({{"range", 3}, {"rangePreview", 1}, {"substringPreview", 2}});
instance->updateIndexTypeStatsOnRegisterCollection(efc2);
expected.range++;
expected.rangePreview++;
expected.substringPreview++;
assertCounters(expected);
auto efc3 = buildConfig({{"suffixPreview", 1}, {"multi", 1}});
instance->updateIndexTypeStatsOnRegisterCollection(efc3);
expected.suffixPreview++;
expected.prefixPreview++;
assertCounters(expected);
instance->updateIndexTypeStatsOnDeregisterCollection(efc2);
expected.range--;
expected.rangePreview--;
expected.substringPreview--;
assertCounters(expected);
instance->updateIndexTypeStatsOnRegisterCollection(efc3);
expected.suffixPreview++;
expected.prefixPreview++;
assertCounters(expected);
instance->updateIndexTypeStatsOnDeregisterCollection(efc3);
expected.suffixPreview--;
expected.prefixPreview--;
assertCounters(expected);
instance->updateIndexTypeStatsOnDeregisterCollection(efc1);
expected.unindexed--;
expected.equality--;
expected.suffixPreview--;
expected.prefixPreview--;
assertCounters(expected);
instance->updateIndexTypeStatsOnDeregisterCollection(efc3);
expected.suffixPreview--;
expected.prefixPreview--;
assertCounters(expected);
}
} // namespace
} // namespace mongo

6
src/mongo/db/catalog/collection_catalog.cpp
View File

@ -45,6 +45,7 @@
#include "mongo/base/status_with.h"
#include "mongo/bson/bsonelement.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/crypto/fle_stats.h"
#include "mongo/db/catalog/collection_options.h"
#include "mongo/db/catalog/collection_record_store_options.h"
#include "mongo/db/catalog/durable_catalog.h"
@ -2233,6 +2234,8 @@ void CollectionCatalog::_registerCollection(OperationContext* opCtx,
}
if (coll->getCollectionOptions().encryptedFieldConfig) {
_stats.queryableEncryption += 1;
FLEStatusSection::get().updateIndexTypeStatsOnRegisterCollection(
coll->getCollectionOptions().encryptedFieldConfig.value());
}
if (isCSFLE1Validator(coll->getValidatorDoc())) {
_stats.csfle += 1;
@ -2305,6 +2308,8 @@ std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(
}
if (coll->getCollectionOptions().encryptedFieldConfig) {
_stats.queryableEncryption -= 1;
FLEStatusSection::get().updateIndexTypeStatsOnDeregisterCollection(
coll->getCollectionOptions().encryptedFieldConfig.value());
}
if (isCSFLE1Validator(coll->getValidatorDoc())) {
_stats.csfle -= 1;
@ -2412,6 +2417,7 @@ void CollectionCatalog::deregisterAllCollectionsAndViews(ServiceContext* svcCtx)
_dropPendingCollection = {};
_stats = {};
FLEStatusSection::get().clearIndexTypeStats();
ResourceCatalog::get().clear();
}