diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f193620ca..5f3515325 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -527,7 +527,7 @@ functions: AWS_ROLE_SESSION_NAME="test" \ .evergreen/run-mongodb-aws-test.sh web-identity - "run oidc auth test with aws credentials": + "run oidc auth test with test credentials": - command: subprocess.exec type: test params: @@ -991,6 +991,30 @@ task_groups: tasks: - oidc-auth-test-azure-latest + - name: testoidc_task_group + setup_group: + - func: fetch source + - func: prepare resources + - func: fix absolute paths + - func: make files executable + - func: "assume ec2 role" + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh + teardown_task: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - oidc-auth-test-latest + - name: test_aws_lambda_task_group setup_group: - func: fetch source @@ -1913,11 +1937,6 @@ tasks: - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - - name: "oidc-auth-test-latest" - commands: - - func: "assume ec2 role" - - func: "run oidc auth test with aws credentials" - - name: load-balancer-test commands: - func: "bootstrap mongo-orchestration" @@ -1927,6 +1946,10 @@ tasks: - func: "run load-balancer" - func: "run tests" + - name: "oidc-auth-test-latest" + commands: + - func: "run oidc auth test with test credentials" + - name: "oidc-auth-test-azure-latest" commands: - command: shell.exec @@ -1940,7 +1963,7 @@ tasks: git commit -m "add files" export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD - export AZUREOIDC_TEST_CMD="source ./env.sh && export OIDC_ENV=azure && ./.evergreen/run-mongodb-oidc-test.sh" + export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "test-fips-standalone" @@ -2974,7 +2997,8 @@ buildvariants: platform: [ rhel8, macos-1100, windows-64-vsMulti-small ] display_name: "MONGODB-OIDC Auth ${platform}" tasks: - - name: "oidc-auth-test-latest" + - name: testoidc_task_group + batchtime: 20160 # 14 days - name: testazureoidc-variant display_name: "Azure OIDC" diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 689c03543..3c045bf7d 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -13,52 +13,11 @@ if [ $OIDC_ENV == "test" ]; then echo "Must specify DRIVERS_TOOLS" exit 1 fi - - # Get the drivers secrets. Use an existing secrets file first. - if [ ! -f "${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh" ]; then - . ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup-secrets.sh - else - source "${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh" - fi - - # Make the OIDC tokens. - set -x - pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc - . ./oidc_get_tokens.sh - popd - - # Set up variables and run the test. - if [ -n "${LOCAL_OIDC_SERVER:-}" ]; then - export MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"} - export MONGODB_URI_SINGLE="${MONGODB_URI}/?authMechanism=MONGODB-OIDC" - export MONGODB_URI_MULTI="${MONGODB_URI}:27018/?authMechanism=MONGODB-OIDC&directConnection=true" - else - set +x # turn off xtrace for this portion - export MONGODB_URI="$OIDC_ATLAS_URI_SINGLE" - export MONGODB_URI_SINGLE="$OIDC_ATLAS_URI_SINGLE/?authMechanism=MONGODB-OIDC" - export MONGODB_URI_MULTI="$OIDC_ATLAS_URI_MULTI/?authMechanism=MONGODB-OIDC" - set -x - fi - export OIDC_TOKEN_FILE="$OIDC_TOKEN_DIR/test_user1" - set +x # turn off xtrace for this portion - export OIDC_ADMIN_USER=$OIDC_ATLAS_USER - export OIDC_ADMIN_PWD=$OIDC_ATLAS_PASSWORD - set -x + source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh elif [ $OIDC_ENV == "azure" ]; then - if [ -z "${AZUREOIDC_RESOURCE:-}" ]; then - echo "Must specify an AZUREOIDC_RESOURCE" - exit 1 - fi - set +x # turn off xtrace for this portion - export OIDC_ADMIN_USER=$AZUREOIDC_USERNAME - export OIDC_ADMIN_PWD=pwd123 - set -x - export MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"} - MONGODB_URI_SINGLE="${MONGODB_URI}/?authMechanism=MONGODB-OIDC" - MONGODB_URI_SINGLE="${MONGODB_URI_SINGLE}&authMechanismProperties=ENVIRONMENT:azure" - export MONGODB_URI_SINGLE="${MONGODB_URI_SINGLE},TOKEN_RESOURCE:${AZUREOIDC_RESOURCE}" - export MONGODB_URI_MULTI=$MONGODB_URI_SINGLE + source ./env.sh + else echo "Unrecognized OIDC_ENV $OIDC_ENV" exit 1 diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 5e5570e48..272851a3b 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -221,7 +221,7 @@ fi if [ -n "$TEST_AUTH_OIDC" ]; then python -m pip install ".[aws]" - TEST_ARGS="test/auth_oidc/test_auth_oidc.py" + TEST_ARGS="test/auth_oidc/test_auth_oidc.py $TEST_ARGS" fi if [ -n "$PERF_TEST" ]; then diff --git a/pymongo/collection.py b/pymongo/collection.py index 2b771f4f6..ceba72aff 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -2372,6 +2372,7 @@ class Collection(common.BaseObject, Generic[_DocumentType]): pipeline, kwargs, explicit_session=session is not None, + comment=comment, user_fields={"cursor": {"firstBatch": 1}}, ) diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index ba00a1101..0fdfc38eb 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -46,7 +46,9 @@ from pymongo.uri_parser import parse_uri ROOT = Path(__file__).parent.parent.resolve() TEST_PATH = ROOT / "auth" / "unified" ENVIRON = os.environ.get("OIDC_ENV", "test") - +DOMAIN = os.environ.get("OIDC_DOMAIN", "") +TOKEN_DIR = os.environ.get("OIDC_TOKEN_DIR", "") +TOKEN_FILE = os.environ.get("OIDC_TOKEN_FILE", "") # Generate unified tests. globals().update(generate_test_classes(str(TEST_PATH), module=__name__)) @@ -56,7 +58,7 @@ class OIDCTestBase(unittest.TestCase): @classmethod def setUpClass(cls): cls.uri_single = os.environ["MONGODB_URI_SINGLE"] - cls.uri_multiple = os.environ["MONGODB_URI_MULTI"] + cls.uri_multiple = os.environ.get("MONGODB_URI_MULTI") cls.uri_admin = os.environ["MONGODB_URI"] def setUp(self): @@ -65,8 +67,10 @@ class OIDCTestBase(unittest.TestCase): def get_token(self, username=None): """Get a token for the current provider.""" if ENVIRON == "test": - token_dir = os.environ["OIDC_TOKEN_DIR"] - token_file = os.path.join(token_dir, username).replace(os.sep, "/") + if username is None: + token_file = TOKEN_FILE + else: + token_file = os.path.join(TOKEN_DIR, username) with open(token_file) as fid: return fid.read() elif ENVIRON == "azure": @@ -95,6 +99,8 @@ class TestAuthOIDCHuman(OIDCTestBase): def setUpClass(cls): if ENVIRON != "test": raise unittest.SkipTest("Human workflows are only tested with the test environment") + if DOMAIN is None: + raise ValueError("Missing OIDC_DOMAIN") super().setUpClass() def create_request_cb(self, username="test_user1", sleep=0): @@ -121,11 +127,14 @@ class TestAuthOIDCHuman(OIDCTestBase): def create_client(self, *args, **kwargs): username = kwargs.get("username", "test_user1") + if kwargs.get("username"): + kwargs["username"] = f"{username}@{DOMAIN}" request_cb = kwargs.pop("request_cb", self.create_request_cb(username=username)) props = kwargs.pop("authmechanismproperties", {"OIDC_HUMAN_CALLBACK": request_cb}) kwargs["retryReads"] = False if not len(args): args = [self.uri_single] + return MongoClient(*args, authmechanismproperties=props, **kwargs) def test_1_1_single_principal_implicit_username(self): @@ -145,6 +154,8 @@ class TestAuthOIDCHuman(OIDCTestBase): client.close() def test_1_3_multiple_principal_user_1(self): + if not self.uri_multiple: + raise unittest.SkipTest("Test Requires Server with Multiple Workflow IdPs") # Create a client with MONGODB_URI_MULTI, a username of test_user1, authMechanism=MONGODB-OIDC, and the OIDC human callback. client = self.create_client(self.uri_multiple, username="test_user1") # Perform a find operation that succeeds. @@ -153,6 +164,8 @@ class TestAuthOIDCHuman(OIDCTestBase): client.close() def test_1_4_multiple_principal_user_2(self): + if not self.uri_multiple: + raise unittest.SkipTest("Test Requires Server with Multiple Workflow IdPs") # Create a human callback that reads in the generated test_user2 token file. # Create a client with MONGODB_URI_MULTI, a username of test_user2, authMechanism=MONGODB-OIDC, and the OIDC human callback. client = self.create_client(self.uri_multiple, username="test_user2") @@ -162,6 +175,8 @@ class TestAuthOIDCHuman(OIDCTestBase): client.close() def test_1_5_multiple_principal_no_user(self): + if not self.uri_multiple: + raise unittest.SkipTest("Test Requires Server with Multiple Workflow IdPs") # Create a client with MONGODB_URI_MULTI, no username, authMechanism=MONGODB-OIDC, and the OIDC human callback. client = self.create_client(self.uri_multiple) # Assert that a find operation fails. @@ -632,15 +647,8 @@ class TestAuthOIDCMachine(OIDCTestBase): def setUp(self): self.request_called = 0 - if ENVIRON == "test": - self.default_username = "test_user1" - else: - self.default_username = None def create_request_cb(self, username=None, sleep=0): - if username is None: - username = self.default_username - def request_token(context): assert isinstance(context.timeout_seconds, int) assert context.version == 1 diff --git a/test/test_index_management.py b/test/test_index_management.py index 25541f980..c9a69aecc 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -25,7 +25,7 @@ sys.path[0:0] = [""] from test import IntegrationTest, unittest from test.unified_format import generate_test_classes -from test.utils import AllowListEventListener +from test.utils import AllowListEventListener, EventListener from pymongo import MongoClient from pymongo.errors import OperationFailure @@ -63,7 +63,9 @@ class TestCreateSearchIndex(IntegrationTest): self.assertIn("arbitraryOption", listener.events[0].command["indexes"][0]) -class TestSearchIndexProse(unittest.TestCase): +class SearchIndexIntegrationBase(unittest.TestCase): + db_name = "test_search_index_base" + @classmethod def setUpClass(cls) -> None: super().setUpClass() @@ -72,9 +74,12 @@ class TestSearchIndexProse(unittest.TestCase): url = os.environ.get("MONGODB_URI") username = os.environ["DB_USER"] password = os.environ["DB_PASSWORD"] - cls.client = MongoClient(url, username=username, password=password) + cls.listener = listener = EventListener() + cls.client = MongoClient( + url, username=username, password=password, event_listeners=[listener] + ) cls.client.drop_database(_NAME) - cls.db = cls.client.test_search_index_prose + cls.db = cls.client[cls.db_name] @classmethod def tearDownClass(cls): @@ -94,6 +99,34 @@ class TestSearchIndexProse(unittest.TestCase): break time.sleep(5) + +class TestSearchIndexIntegration(SearchIndexIntegrationBase): + db_name = "test_search_index" + + def test_comment_field(self): + # Create a collection with the "create" command using a randomly generated name (referred to as ``coll0``). + coll0 = self.db[f"col{uuid.uuid4()}"] + coll0.insert_one({}) + + # Create a new search index on ``coll0`` that implicitly passes its type. + search_definition = {"mappings": {"dynamic": False}} + self.listener.reset() + implicit_search_resp = coll0.create_search_index( + model={"name": _NAME + "-implicit", "definition": search_definition}, comment="foo" + ) + event = self.listener.events[0] + self.assertEqual(event.command["comment"], "foo") + + # Get the index definition. + self.listener.reset() + coll0.list_search_indexes(name=implicit_search_resp, comment="foo").next() + event = self.listener.events[0] + self.assertEqual(event.command["comment"], "foo") + + +class TestSearchIndexProse(SearchIndexIntegrationBase): + db_name = "test_search_index_prose" + def test_case_1(self): """Driver can successfully create and list search indexes.""" diff --git a/test/unified_format.py b/test/unified_format.py index 2a44d530a..86056278a 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1062,8 +1062,8 @@ class UnifiedSpecTestMixinV1(IntegrationTest): self.skipTest("PyMongo's open_download_stream does not cap the stream's lifetime") if "unpin after TransientTransactionError error on" in spec["description"]: - if client_context.version[0] == 8: - self.skipTest("Skipping TransientTransactionError pending PYTHON-4182") + self.skipTest("Skipping TransientTransactionError pending PYTHON-4227") + if "unpin after non-transient error on abort" in spec["description"]: if client_context.version[0] == 8: self.skipTest("Skipping TransientTransactionError pending PYTHON-4182") @@ -1252,7 +1252,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): clients = self.mongos_clients if self.mongos_clients else [self.client] for client in clients: try: - self.client.admin.command("killAllSessions", []) + client.admin.command("killAllSessions", []) except OperationFailure: # "operation was interrupted" by killing the command's # own session.