From 4863cb9653d068b23e7c45281c47bcada8c4568e Mon Sep 17 00:00:00 2001 From: Didier Nadeau Date: Thu, 6 Feb 2025 13:38:29 -0800 Subject: [PATCH] SERVER-85804 Support Proxy Protocol on Mongod (#31814) Co-authored-by: Sara Golemon GitOrigin-RevId: ab2539a7da36e7b87df38e8853174f7facfc838c --- .../fully_disabled_feature_flags.yml | 2 +- .../network/mongod_proxy_protocol.js | 102 ++++++++++++++++++ src/mongo/db/mongod_main.cpp | 38 +++---- src/mongo/db/repl/replication_info.cpp | 4 + src/mongo/db/server_feature_flags.idl | 4 +- .../transport/asio/asio_session_impl.cpp | 9 ++ .../proxy_protocol_header_parser.cpp | 3 + .../transport/proxy_protocol_header_parser.h | 10 ++ 8 files changed, 150 insertions(+), 22 deletions(-) create mode 100644 jstests/noPassthrough/network/mongod_proxy_protocol.js diff --git a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml index fc63e14f5f6..e8100c794bb 100644 --- a/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml +++ b/buildscripts/resmokeconfig/fully_disabled_feature_flags.yml @@ -17,5 +17,5 @@ - featureFlagEgressGrpcForSearch - featureFlagTrackUnshardedCollectionsUponCreation - featureFlagTSBucketingParametersUnchanged -- featureFlagMongogProxyProcolSupport +- featureFlagMongodProxyProcolSupport - featureFlagShardAuthoritativeDbMetadata diff --git a/jstests/noPassthrough/network/mongod_proxy_protocol.js b/jstests/noPassthrough/network/mongod_proxy_protocol.js new file mode 100644 index 00000000000..79cc4b9ee4b --- /dev/null +++ b/jstests/noPassthrough/network/mongod_proxy_protocol.js @@ -0,0 +1,102 @@ +/** + * Verify mongod support proxy protocol connections. + * @tags: [ + * requires_fcv_81, + * # TODO (SERVER-97257): Re-enable this test or add an explanation why it is incompatible. + * embedded_router_incompatible, + * grpc_incompatible, + * ] + */ + +if (_isWindows()) { + quit(); +} +import {ProxyProtocolServer} from "jstests/sharding/libs/proxy_protocol.js"; +import {ReplSetTest} from "jstests/libs/replsettest.js"; + +function runHello(port, loadBalanced) { + let uri = `mongodb://127.0.0.1:${port}`; + if (typeof loadBalanced != 'undefined') { + uri += `/?loadBalanced=${loadBalanced}`; + } + const conn = new Mongo(uri); + assert.neq(null, conn, 'Client was unable to connect to the load balancer port'); + assert.commandWorked(conn.getDB('admin').runCommand({hello: 1})); +} + +function failInvalidProtocol(node, port, id, attrs, loadBalanced, count) { + let uri = `mongodb://127.0.0.1:${port}`; + if (typeof loadBalanced != 'undefined') { + uri += `/?loadBalanced=${loadBalanced}`; + } + try { + new Mongo(uri); + assert(false, 'Client was unable to connect to the load balancer port'); + } catch (err) { + assert(checkLog.checkContainsWithCountJson(node, id, attrs, count, undefined, true), + `Did not find log id ${tojson(id)} with attr ${tojson(attrs)} ${ + tojson(id)} times in the log`); + } +} + +// Test that you can connect to the load balancer port over a proxy. +function testProxyProtocolReplicaSet(ingressPort, egressPort, version) { + let proxy_server = new ProxyProtocolServer(ingressPort, egressPort, version); + proxy_server.start(); + + let rs = new ReplSetTest({nodes: 1, nodeOptions: {"proxyPort": egressPort}}); + rs.startSet({setParameter: {featureFlagMongodProxyProcolSupport: true}}); + rs.initiate(); + + // Connecting to the to the proxy port succeeds. + runHello(ingressPort, undefined); + runHello(ingressPort, false); + + // Connecting to the to the proxy port with {loadBalanced: true} fails. + const lbmismatch = { + "error": "LoadBalancerSupportMismatch: Mongod does not support load-balanced connections" + }; + + const kCmdExecAssertion = 21962; + const node = rs.getPrimary(); + failInvalidProtocol(node, ingressPort, kCmdExecAssertion, lbmismatch, "true", 1); + + // Connecting to the standard port without proxy header succeeds. + const port = node.port; + runHello(port, undefined); + runHello(port, false); + + // Connecting to the standard port without and with {loadBalanced:true} proxy header fails. + failInvalidProtocol(node, port, kCmdExecAssertion, lbmismatch, "true", 2); + + // Connecting to the proxy port without proxy header fails. + const kProxyProtocolParseError = 6067900; + failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, "true", 1); + failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, "false", 2); + failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, undefined, 3); + + proxy_server.stop(); + + // Connecting to the standard port with proxy header fails. + proxy_server = new ProxyProtocolServer(ingressPort, port, version); + proxy_server.start(); + const attrs = { + "error": { + "code": ErrorCodes.OperationFailed, + "codeName": "OperationFailed", + "errmsg": "ProxyProtocol message detected on mongorpc port", + } + }; + failInvalidProtocol(node, ingressPort, 22988, attrs, "true", 1); + failInvalidProtocol(node, ingressPort, 22988, attrs, "false", 2); + failInvalidProtocol(node, ingressPort, 22988, attrs, undefined, 3); + proxy_server.stop(); + + rs.stopSet(); +} + +const ingressPort = allocatePort(); +const egressPort = allocatePort(); + +testProxyProtocolReplicaSet(ingressPort, egressPort, 1); +testProxyProtocolReplicaSet(ingressPort, egressPort, 2); diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp index 025e7e19ffb..91e6294628f 100644 --- a/src/mongo/db/mongod_main.cpp +++ b/src/mongo/db/mongod_main.cpp @@ -188,6 +188,7 @@ #include "mongo/db/s/sharding_initialization_mongod.h" #include "mongo/db/s/sharding_ready.h" #include "mongo/db/s/transaction_coordinator_service.h" +#include "mongo/db/server_feature_flags_gen.h" #include "mongo/db/server_lifecycle_monitor.h" #include "mongo/db/server_options.h" #include "mongo/db/service_context.h" @@ -322,7 +323,6 @@ const ntservice::NtServiceDefaultStrings defaultServiceStrings = { auto makeTransportLayer(ServiceContext* svcCtx) { boost::optional routerPort; - boost::optional loadBalancerPort; boost::optional proxyPort; if (serverGlobalParams.routerPort) { @@ -336,20 +336,23 @@ auto makeTransportLayer(ServiceContext* svcCtx) { // TODO SERVER-78730: add support for load-balanced connections. } - if (serverGlobalParams.proxyPort) { - proxyPort = *serverGlobalParams.proxyPort; - if (*proxyPort == serverGlobalParams.port) { - LOGV2_ERROR(9967800, - "The proxy port must be different from the public listening port.", - "port"_attr = serverGlobalParams.port); - quickExit(ExitCode::badOptions); - } + // (Ignore FCV check): The proxy port needs to be open before the FCV is set. + if (gFeatureFlagMongodProxyProcolSupport.isEnabledAndIgnoreFCVUnsafe()) { + if (serverGlobalParams.proxyPort) { + proxyPort = *serverGlobalParams.proxyPort; + if (*proxyPort == serverGlobalParams.port) { + LOGV2_ERROR(9967800, + "The proxy port must be different from the public listening port.", + "port"_attr = serverGlobalParams.port); + quickExit(ExitCode::badOptions); + } - if (routerPort && *proxyPort == *routerPort) { - LOGV2_ERROR(9967801, - "The proxy port must be different from the public router port.", - "port"_attr = *routerPort); - quickExit(ExitCode::badOptions); + if (routerPort && *proxyPort == *routerPort) { + LOGV2_ERROR(9967801, + "The proxy port must be different from the public router port.", + "port"_attr = *routerPort); + quickExit(ExitCode::badOptions); + } } } @@ -369,11 +372,8 @@ auto makeTransportLayer(ServiceContext* svcCtx) { } #endif - return transport::TransportLayerManagerImpl::createWithConfig(&serverGlobalParams, - svcCtx, - useEgressGRPC, - std::move(loadBalancerPort), - std::move(routerPort)); + return transport::TransportLayerManagerImpl::createWithConfig( + &serverGlobalParams, svcCtx, useEgressGRPC, std::move(proxyPort), std::move(routerPort)); } ExitCode initializeTransportLayer(ServiceContext* serviceContext, BSONObjBuilder* timerReport) { diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp index 7ccc86ac29d..5cd8154b91a 100644 --- a/src/mongo/db/repl/replication_info.cpp +++ b/src/mongo/db/repl/replication_info.cpp @@ -413,6 +413,10 @@ public: auto cmd = idl::parseCommandDocument( IDLParserContext("hello", vts, dbName.tenantId(), sc), cmdObj); + uassert(ErrorCodes::LoadBalancerSupportMismatch, + "Mongod does not support load-balanced connections", + !cmd.getLoadBalanced().value_or(false)); + shardWaitInHello.execute( [&](const BSONObj& customArgs) { _handleHelloFailPoint(customArgs, opCtx, cmdObj); }); diff --git a/src/mongo/db/server_feature_flags.idl b/src/mongo/db/server_feature_flags.idl index 25a7b32bdf9..b4e8122682e 100644 --- a/src/mongo/db/server_feature_flags.idl +++ b/src/mongo/db/server_feature_flags.idl @@ -134,9 +134,9 @@ feature_flags: cpp_varname: gFeatureFlagExposeClientIpInAuditLogs default: false shouldBeFCVGated: true - featureFlagMongogProxyProcolSupport: + featureFlagMongodProxyProcolSupport: description: "Enables non-OCS proxy protocol connections on Mongos and Mongod" - cpp_varname: gFeatureFlagMongogProxyProcolSupport + cpp_varname: gFeatureFlagMongodProxyProcolSupport default: false shouldBeFCVGated: true featureFlagRawDataCrudOperations: diff --git a/src/mongo/transport/asio/asio_session_impl.cpp b/src/mongo/transport/asio/asio_session_impl.cpp index fd83131e2f6..d3f36c55292 100644 --- a/src/mongo/transport/asio/asio_session_impl.cpp +++ b/src/mongo/transport/asio/asio_session_impl.cpp @@ -747,6 +747,15 @@ Future CommonAsioSession::maybeHandshakeSSLForIngress(const MutableBufferS if (checkForHTTPRequest(buffer)) { return Future::makeReady(false); } + + if (maybeProxyProtocolHeader( + StringData(asio::buffer_cast(buffer), asio::buffer_size(buffer)))) { + // Protocol requirements mean that neither raw mongorpc nor TLS client hello will look like + // Proxy. + return Future::makeReady( + Status(ErrorCodes::OperationFailed, "ProxyProtocol message detected on mongorpc port")); + } + // This logic was taken from the old mongo/util/net/sock.cpp. // // It lets us run both TLS and unencrypted mongo over the same port. diff --git a/src/mongo/transport/proxy_protocol_header_parser.cpp b/src/mongo/transport/proxy_protocol_header_parser.cpp index 89acfcecfc5..6137c2164ca 100644 --- a/src/mongo/transport/proxy_protocol_header_parser.cpp +++ b/src/mongo/transport/proxy_protocol_header_parser.cpp @@ -455,5 +455,8 @@ boost::optional parseProxyProtocolHeader(StringData buffer) { } } +bool maybeProxyProtocolHeader(StringData buffer) { + return buffer.startsWith(kV1Start) || buffer.startsWith(kV2Start); +} } // namespace mongo::transport diff --git a/src/mongo/transport/proxy_protocol_header_parser.h b/src/mongo/transport/proxy_protocol_header_parser.h index 9e81586910a..0fcc599bc27 100644 --- a/src/mongo/transport/proxy_protocol_header_parser.h +++ b/src/mongo/transport/proxy_protocol_header_parser.h @@ -86,6 +86,16 @@ struct ParserResults { */ boost::optional parseProxyProtocolHeader(StringData buffer); +/** + * Peek a buffer fo at least 12 bytes to determine if it may be a proxy protocol header. + * + * Note that this does not definitively identify the initial packet as proxy protocol, + * it only establishes that it is possible that it is such. + * To be used in determining appropriate error messages during otherwise failed + * initial handshakes only. + */ +bool maybeProxyProtocolHeader(StringData buffer); + namespace proxy_protocol_details { static constexpr size_t kMaxUnixPathLength = 108;