mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-108051 Adjust FLE2 substringPreview per-field max length limit (#39322)

Browse Source 
GitOrigin-RevId: 0417aa53f9bcada88661221e8168508db89112e9
This commit is contained in:
Erwin Pe 2025-07-31 09:08:43 -04:00 committed by MongoDB Bot
parent 4909b2b67b
commit 213fce2e3e
8 changed files with 186 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/mongo/crypto/encryption_fields_validation.cpp
View File

@ -48,10 +48,8 @@
#include "mongo/crypto/fle_numeric.h"
#include "mongo/db/field_ref.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/server_feature_flags_gen.h"
#include "mongo/stdx/unordered_set.h"
#include "mongo/util/assert_util.h"
#include "mongo/util/fail_point.h"
#include "mongo/util/str.h"
#include "mongo/util/uuid.h"
@ -62,7 +60,6 @@
namespace mongo {
MONGO_FAIL_POINT_DEFINE(allowOutOfBoundsSubstringParameters);
Value coerceValueToRangeIndexTypes(Value val, BSONType fieldType) {
BSONType valType = val.getType();
@ -486,35 +483,6 @@ void validateTextSearchIndex(BSONType fieldType,
uassert(9783408,
"strMaxQueryLength cannot be greater than strMaxLength",
query.getStrMaxQueryLength().value() <= query.getStrMaxLength().value());
// Substring specifically strictly bounds strMinQueryLength to >= 2, strMaxQueryLength to <=
// 10, and strMaxLength to <= 400.
uassert(10453200,
fmt::format("strMinQueryLength ({}) must be >= 2 and <= strMaxQueryLength ({}) for "
"{} query type of field {}",
query.getStrMinQueryLength().value(),
query.getStrMaxQueryLength().value(),
qTypeStr,
fieldPath),
MONGO_unlikely(allowOutOfBoundsSubstringParameters.shouldFail()) ||
query.getStrMinQueryLength().value() >= 2);
uassert(10453201,
fmt::format("strMaxQueryLength ({}) must be <= 10 and >= strMinQueryLength ({}) "
"for {} query type of field {}",
query.getStrMaxQueryLength().value(),
query.getStrMinQueryLength().value(),
qTypeStr,
fieldPath),
MONGO_unlikely(allowOutOfBoundsSubstringParameters.shouldFail()) ||
query.getStrMaxQueryLength().value() <= 10);
uassert(10453202,
fmt::format("strMaxLength ({}) must be <= 400 "
"{} query type of field {}",
query.getStrMaxLength().value(),
qTypeStr,
fieldPath),
MONGO_unlikely(allowOutOfBoundsSubstringParameters.shouldFail()) ||
query.getStrMaxLength().value() <= 400);
}
if (previousCaseSensitivity.has_value() &&

25
src/mongo/crypto/encryption_fields_validation_test.cpp
View File

@ -537,31 +537,6 @@ TEST(FLEValidationUtils, ValidateTextSearchIndexSubstring) {
BSONType::string, "foo"_sd, qtc, boost::none, boost::none, boost::none),
AssertionException,
9783408);
// min query length < 2
qtc.setStrMaxLength(400);
qtc.setStrMinQueryLength(1);
qtc.setStrMaxQueryLength(10);
ASSERT_THROWS_CODE(validateTextSearchIndex(
BSONType::string, "foo", qtc, boost::none, boost::none, boost::none),
AssertionException,
10453200);
// max query length > 10
qtc.setStrMaxLength(400);
qtc.setStrMinQueryLength(2);
qtc.setStrMaxQueryLength(11);
ASSERT_THROWS_CODE(validateTextSearchIndex(
BSONType::string, "foo", qtc, boost::none, boost::none, boost::none),
AssertionException,
10453201);
// max length > 400
qtc.setStrMaxLength(401);
qtc.setStrMinQueryLength(2);
qtc.setStrMaxQueryLength(10);
ASSERT_THROWS_CODE(validateTextSearchIndex(
BSONType::string, "foo", qtc, boost::none, boost::none, boost::none),
AssertionException,
10453202);
// Make sure valid configuration passes.
qtc.setStrMaxLength(400);
qtc.setStrMinQueryLength(2);

68
src/mongo/crypto/fle_crypto.cpp
View File

@ -3966,12 +3966,78 @@ void EncryptionInformationHelpers::checkTagLimitsAndStorageNotExceeded(
uassert(
10431800,
fmt::format("Cannot create a collection where the worst case total Queryable Encryption "
"tag storage size ({}) exceeds the max BSON size ({})",
"tag storage size ({}) exceeds the max BSON size ({}). Consider reducing the "
"number of encrypted fields in the schema, or tuning the indexing parameters.",
totalTagStorage,
BSONObjMaxUserSize),
shouldOverrideTotalTagOverheadLimit || totalTagStorage <= BSONObjMaxUserSize);
}
void EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceeded(
const EncryptedFieldConfig& ef) {
static_assert(kSubstringPreviewLowerBoundMin <= kSubstringPreviewUpperBoundMax);
static_assert(kSubstringPreviewUpperBoundMax <= kSubstringPreviewMaxLengthMax);
static constexpr StringData bypassMsg =
"Consider setting the fleDisableSubstringPreviewParameterLimits cluster parameter to true "
"to bypass this limit.";
if (ServerParameterSet::getClusterParameterSet()
->get<ClusterParameterWithStorage<FLEOverrideSubstringPreviewLimits>>(
"fleDisableSubstringPreviewParameterLimits")
->getValue(boost::none)
.getShouldOverride()) {
return;
}
auto checkOneQueryType = [](StringData path, const QueryTypeConfig& qtc) {
if (qtc.getQueryType() != QueryTypeEnum::SubstringPreview) {
return;
}
int32_t ub = qtc.getStrMaxQueryLength().get();
int32_t lb = qtc.getStrMinQueryLength().get();
int32_t max = qtc.getStrMaxLength().get();
uassert(10453200,
fmt::format("strMinQueryLength ({}) must be >= {} for substringPreview query "
"type of field {}. {}",
lb,
kSubstringPreviewLowerBoundMin,
path,
bypassMsg),
lb >= kSubstringPreviewLowerBoundMin);
uassert(10453201,
fmt::format("strMaxQueryLength ({}) must be >= {} for substringPreview query "
"type of field {}. {}",
ub,
kSubstringPreviewUpperBoundMax,
path,
bypassMsg),
ub <= kSubstringPreviewUpperBoundMax);
uassert(10453202,
fmt::format("strMaxLength ({}) must be >= {} for substringPreview query "
"type of field {}. {}",
max,
kSubstringPreviewMaxLengthMax,
path,
bypassMsg),
max <= kSubstringPreviewMaxLengthMax);
};
for (const auto& field : ef.getFields()) {
if (!field.getQueries()) {
continue;
}
visit(
OverloadedVisitor{[&](QueryTypeConfig qtc) { checkOneQueryType(field.getPath(), qtc); },
[&](std::vector<QueryTypeConfig> queries) {
for (auto& qtc : queries) {
checkOneQueryType(field.getPath(), qtc);
}
}},
field.getQueries().get());
}
}
std::pair<EncryptedBinDataType, ConstDataRange> fromEncryptedConstDataRange(ConstDataRange cdr) {
ConstDataRangeCursor cdrc(cdr);

21
src/mongo/crypto/fle_crypto.h
View File

@ -1088,6 +1088,27 @@ public:
static constexpr uint32_t kFLE2PerTagStorageBytes =
sizeof(FLE2TagAndEncryptedMetadataBlock::SerializedBlob) + sizeof(PrfBlock);
static void checkTagLimitsAndStorageNotExceeded(const EncryptedFieldConfig& ef);
/**
* Soft limits for substringPreview parameters. These can be bypassed
* through setParameter fleDisableSubstringPreviewParameterLimits.
*/
static constexpr int32_t kSubstringPreviewLowerBoundMin = 2;
static constexpr int32_t kSubstringPreviewUpperBoundMax = 10;
static constexpr int32_t kSubstringPreviewMaxLengthMax = 60;
/**
* checkSubstringPreviewParameterLimitsNotExceeded throws if
* fleDisableSubstringPreviewParameterLimits is false and either of the following conditions
* are met:
* 1. There exists a substringPreview field in EncryptedFieldConfig with strMinQueryLength
* less than kSubstringPreviewLowerBoundMin.
* 2. There exists a substringPreview field in EncryptedFieldConfig with strMaxQueryLength
* greater than kSubstringPreviewUpperBoundMax.
* 3. There exists a substringPreview field in EncryptedFieldConfig with strMaxLength
* greater than kSubstringPreviewMaxLengthMax.
*/
static void checkSubstringPreviewParameterLimitsNotExceeded(const EncryptedFieldConfig& ef);
};
/**

89
src/mongo/crypto/fle_crypto_test.cpp
View File

@ -2698,39 +2698,28 @@ TEST_F(ServiceContextTest, EncryptionInformation_TestTagLimitsForTextSearch) {
// substring field under limit
std::vector<QueryTypeConfig> qtc = {
makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 2, 10, 200)};
assertExpectedMaxTags(qtc, 1756);
makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 10, 100, 900)};
assertExpectedMaxTags(qtc, 76987);
doOneFieldTest(qtc, {});
// substring tag limit tests require a failpoint to circumvent parameter limits
{
FailPointEnableBlock fp("allowOutOfBoundsSubstringParameters");
// substring field at limit
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, 1, 83'999);
assertExpectedMaxTags(qtc, 84'000);
doOneFieldTest(qtc, {});
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, 2, 42'000);
assertExpectedMaxTags(qtc, 84'000);
doOneFieldTest(qtc, {});
// substring field over limit
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 10, 100, 1000);
assertExpectedMaxTags(qtc, 86'087);
doOneFieldTest(qtc, 10384602);
// overflow uint32_t
qtc.front() =
makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, INT32_MAX, INT32_MAX);
doOneFieldTest(qtc, 10384601);
}
// substring field should still be under tag limit even with min allowed lb and max allowed ub
// and mlen
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 2, 10, 400);
assertExpectedMaxTags(qtc, 3556);
// substring field at limit
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, 1, 83'999);
assertExpectedMaxTags(qtc, 84'000);
doOneFieldTest(qtc, {});
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, 2, 42'000);
assertExpectedMaxTags(qtc, 84'000);
doOneFieldTest(qtc, {});
// substring field over limit
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 10, 100, 1000);
assertExpectedMaxTags(qtc, 86'087);
doOneFieldTest(qtc, 10384602);
// overflow uint32_t
qtc.front() = makeTextQueryTypeConfig(QueryTypeEnum::SubstringPreview, 1, INT32_MAX, INT32_MAX);
doOneFieldTest(qtc, 10384601);
for (auto qtype : {QueryTypeEnum::SuffixPreview, QueryTypeEnum::PrefixPreview}) {
// suffix/prefix field under limit
qtc.front() = makeTextQueryTypeConfig(qtype, 9, 109, {});
@ -2945,6 +2934,48 @@ TEST_F(ServiceContextTest, EncryptionInformation_TestTagStorageLimits) {
doMultipleFieldsTest(efc, {});
}
TEST_F(ServiceContextTest, EncryptionInformation_TestSubstringPreviewParameterLimits) {
auto makeQueryTypeConfig = [](int32_t lb, int32_t ub, int32_t mlen) {
QueryTypeConfig qtc{QueryTypeEnum::SubstringPreview};
qtc.setStrMinQueryLength(lb);
qtc.setStrMaxQueryLength(ub);
qtc.setStrMaxLength(mlen);
return qtc;
};
auto doOneFieldTest = [](const std::vector<QueryTypeConfig>& qtc, boost::optional<int> error) {
EncryptedFieldConfig efc;
EncryptedField field{UUID::gen(), "field"};
field.setBsonType("string"_sd);
field.setQueries(std::variant<std::vector<QueryTypeConfig>, QueryTypeConfig>{qtc});
efc.setFields({field});
if (error) {
ASSERT_THROWS_CODE(
EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceeded(efc),
DBException,
*error);
} else {
ASSERT_DOES_NOT_THROW(
EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceeded(efc));
}
};
// substring max length over upper limit
std::vector<QueryTypeConfig> qtc = {makeQueryTypeConfig(2, 10, 900)};
doOneFieldTest(qtc, 10453202);
// substring max query length over upper limit
qtc[0] = makeQueryTypeConfig(2, 60, 60);
doOneFieldTest(qtc, 10453201);
// substring min query length below lower limit
qtc[0] = makeQueryTypeConfig(1, 10, 60);
doOneFieldTest(qtc, 10453200);
// substring params all within limtis
qtc[0] = makeQueryTypeConfig(2, 10, 60);
doOneFieldTest(qtc, {});
}
TEST_F(ServiceContextTest, IndexedFields_FetchTwoLevels) {
TestKeyVault keyVault;

24
src/mongo/crypto/fle_options.idl
View File

@ -87,6 +87,18 @@ structs:
type: bool
default: false
FLEOverrideSubstringPreviewLimits:
description: "Container for the fleDisableSubstringPreviewParameterLimits CSP"
strict: false
inline_chained_structs: true
chained_structs:
ClusterServerParameter: clusterServerParameter
fields:
shouldOverride:
description: "Flag indicating whether or not the substring preview limits should be overridden"
type: bool
default: false
server_parameters:
unsupportedDangerousTestingFLEDiagnosticsEnabled:
description: "Start with test-only FLE statistics behavior enabled"
@ -117,3 +129,15 @@ server_parameters:
min_fcv: 8.2
redact: false
omit_in_ftdc: false
fleDisableSubstringPreviewParameterLimits:
description:
"If set to true, allows QE collection creation to succeed if there exists
substringPreview fields whose parameters exceed the preview soft limits."
set_at: cluster
cpp_varname: "fleOverrideParameterLimits"
cpp_vartype: FLEOverrideSubstringPreviewLimits
condition:
min_fcv: 8.2
redact: false
omit_in_ftdc: false

2
src/mongo/db/commands/create_command.cpp
View File

@ -401,6 +401,8 @@ public:
} else {
EncryptionInformationHelpers::checkTagLimitsAndStorageNotExceeded(
cmd.getEncryptedFields().get());
EncryptionInformationHelpers::checkSubstringPreviewParameterLimitsNotExceeded(
cmd.getEncryptedFields().get());
}
}

24
src/mongo/db/fle_crud_test.cpp
View File

@ -2205,9 +2205,9 @@ void QETextSearchCrudTest::doInsertsAndVerifyExpectations(
TEST_F(QETextSearchCrudTest, BasicSubstring) {
addSchema({.type = QueryTypeEnum::SubstringPreview,
.lb = 2,
.ub = 10,
.mlen = 400,
.lb = 10,
.ub = 100,
.mlen = 1000,
.casef = false,
.diacf = false});
doInsertsAndVerifyExpectations({{"demonstration", "demonstration"}});
@ -2247,9 +2247,9 @@ TEST_F(QETextSearchCrudTest, BasicPrefixAndSuffix) {
TEST_F(QETextSearchCrudTest, RepeatingSubstring) {
addSchema({.type = QueryTypeEnum::SubstringPreview,
.lb = 2,
.ub = 10,
.mlen = 400,
.lb = 10,
.ub = 100,
.mlen = 1000,
.casef = false,
.diacf = false});
doInsertsAndVerifyExpectations({{"aaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaa"}});
@ -2268,9 +2268,9 @@ TEST_F(QETextSearchCrudTest, FoldAsciiSuffix) {
TEST_F(QETextSearchCrudTest, UnicodeSubstring) {
addSchema({.type = QueryTypeEnum::SubstringPreview,
.lb = 2,
.lb = 1,
.ub = 5,
.mlen = 400,
.mlen = 1000,
.casef = false,
.diacf = false});
doInsertsAndVerifyExpectations({{"私はぺんです.", "私はぺんです."}});
@ -2280,7 +2280,7 @@ TEST_F(QETextSearchCrudTest, FoldUnicodeSubstring) {
addSchema({.type = QueryTypeEnum::SubstringPreview,
.lb = 3,
.ub = 5,
.mlen = 400,
.mlen = 1000,
.casef = true,
.diacf = true});
doInsertsAndVerifyExpectations(
@ -2290,9 +2290,9 @@ TEST_F(QETextSearchCrudTest, FoldUnicodeSubstring) {
TEST_F(QETextSearchCrudTest, BasicSubstringMultipleInserts) {
addSchema({.type = QueryTypeEnum::SubstringPreview,
.lb = 2,
.ub = 10,
.mlen = 400,
.lb = 10,
.ub = 100,
.mlen = 1000,
.casef = false,
.diacf = false});
doInsertsAndVerifyExpectations({{"demonstration", "demonstration"},