SERVER-123240 Ensure that polls to serverStatus in the egress connection establishment rate limiter test account for a constantly-running server (#51003)
GitOrigin-RevId: c4cca71bc8e0b0bd3aa775670242b255de0cf821
This commit is contained in:
parent
06ecce7def
commit
f2e9b16c06
@ -107,6 +107,21 @@ public:
|
||||
<< false));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current establishmentRateLimit section from serverStatus, or an empty BSONObj
|
||||
* if not present.
|
||||
*/
|
||||
BSONObj getRateLimiterStats() {
|
||||
auto serverStatus = runSetupCommandSync(DatabaseName::kAdmin, BSON("serverStatus" << 1));
|
||||
if (serverStatus.hasField("connections")) {
|
||||
auto connections = serverStatus["connections"].Obj();
|
||||
if (connections.hasField("establishmentRateLimit")) {
|
||||
return connections["establishmentRateLimit"].Obj().getOwned();
|
||||
}
|
||||
}
|
||||
return BSONObj();
|
||||
}
|
||||
|
||||
void waitForRateLimiterStat(std::function<bool(const BSONObj&)> pred, StringData description) {
|
||||
const auto deadline = Date_t::now() + Seconds{30};
|
||||
while (Date_t::now() < deadline) {
|
||||
@ -197,13 +212,17 @@ TEST_F(EgressPoolRateLimiterResilienceTest, RejectionWithEstablishedConnection)
|
||||
SKIP_ON_GRPC_RATE_LIMITER();
|
||||
|
||||
withEstablishedConnection([&] {
|
||||
auto baseline = getRateLimiterStats();
|
||||
auto baselineRejected = baseline.isEmpty() ? 0 : baseline["rejected"].numberLong();
|
||||
|
||||
enableRateLimiter(/*maxQueueDepth=*/1);
|
||||
auto hangFP = configureFailPoint("hangInRateLimiter", BSONObj());
|
||||
|
||||
auto futures = sendPings(6, Minutes{10});
|
||||
|
||||
waitForRateLimiterStat([](const BSONObj& rl) { return rl["rejected"].numberLong() >= 1; },
|
||||
"rejected >= 1");
|
||||
waitForRateLimiterStat(
|
||||
[&](const BSONObj& rl) { return rl["rejected"].numberLong() > baselineRejected; },
|
||||
"rejected increased from baseline");
|
||||
assertPoolHasEstablishedConnection(
|
||||
"Pool should still have >= 1 established connection after rejections");
|
||||
|
||||
@ -220,16 +239,20 @@ TEST_F(EgressPoolRateLimiterResilienceTest, TimeoutWithEstablishedConnection) {
|
||||
SKIP_ON_GRPC_RATE_LIMITER();
|
||||
|
||||
withEstablishedConnection([&] {
|
||||
auto baseline = getRateLimiterStats();
|
||||
auto baselineInterrupted =
|
||||
baseline.isEmpty() ? 0 : baseline["interruptedDueToClientDisconnect"].numberLong();
|
||||
|
||||
enableRateLimiter(/*maxQueueDepth=*/10);
|
||||
auto hangFP = configureFailPoint("hangInRateLimiter", BSONObj());
|
||||
|
||||
auto futures = sendPings(3, Minutes{10});
|
||||
|
||||
waitForRateLimiterStat(
|
||||
[](const BSONObj& rl) {
|
||||
return rl["interruptedDueToClientDisconnect"].numberLong() >= 1;
|
||||
[&](const BSONObj& rl) {
|
||||
return rl["interruptedDueToClientDisconnect"].numberLong() > baselineInterrupted;
|
||||
},
|
||||
"interruptedDueToClientDisconnect >= 1");
|
||||
"interruptedDueToClientDisconnect increased from baseline");
|
||||
assertPoolHasEstablishedConnection(
|
||||
"Pool should still have >= 1 established connection after queue timeouts");
|
||||
|
||||
@ -245,13 +268,17 @@ TEST_F(EgressPoolRateLimiterResilienceTest, TimeoutWithEstablishedConnection) {
|
||||
TEST_F(EgressPoolRateLimiterResilienceTest, RejectionWithNoEstablishedConnection) {
|
||||
SKIP_ON_GRPC_RATE_LIMITER();
|
||||
|
||||
auto baseline = getRateLimiterStats();
|
||||
auto baselineRejected = baseline.isEmpty() ? 0 : baseline["rejected"].numberLong();
|
||||
|
||||
enableRateLimiter(/*maxQueueDepth=*/1);
|
||||
auto hangFP = configureFailPoint("hangInRateLimiter", BSONObj());
|
||||
|
||||
auto futures = sendPings(6, Seconds{30});
|
||||
|
||||
waitForRateLimiterStat([](const BSONObj& rl) { return rl["rejected"].numberLong() >= 1; },
|
||||
"rejected >= 1");
|
||||
waitForRateLimiterStat(
|
||||
[&](const BSONObj& rl) { return rl["rejected"].numberLong() > baselineRejected; },
|
||||
"rejected increased from baseline");
|
||||
|
||||
hangFP.disable();
|
||||
disableRateLimiter();
|
||||
@ -264,14 +291,20 @@ TEST_F(EgressPoolRateLimiterResilienceTest, RejectionWithNoEstablishedConnection
|
||||
TEST_F(EgressPoolRateLimiterResilienceTest, TimeoutWithNoEstablishedConnection) {
|
||||
SKIP_ON_GRPC_RATE_LIMITER();
|
||||
|
||||
auto baseline = getRateLimiterStats();
|
||||
auto baselineInterrupted =
|
||||
baseline.isEmpty() ? 0 : baseline["interruptedDueToClientDisconnect"].numberLong();
|
||||
|
||||
enableRateLimiter(/*maxQueueDepth=*/10);
|
||||
auto hangFP = configureFailPoint("hangInRateLimiter", BSONObj());
|
||||
|
||||
auto futures = sendPings(3, Seconds{30});
|
||||
|
||||
waitForRateLimiterStat(
|
||||
[](const BSONObj& rl) { return rl["interruptedDueToClientDisconnect"].numberLong() >= 1; },
|
||||
"interruptedDueToClientDisconnect >= 1");
|
||||
[&](const BSONObj& rl) {
|
||||
return rl["interruptedDueToClientDisconnect"].numberLong() > baselineInterrupted;
|
||||
},
|
||||
"interruptedDueToClientDisconnect increased from baseline");
|
||||
|
||||
hangFP.disable();
|
||||
disableRateLimiter();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user