From e80141ed1c3afb9edf97f2a3a7507f78ebb9a97f Mon Sep 17 00:00:00 2001 From: Julius Park Date: Mon, 8 Nov 2021 16:19:24 -0800 Subject: [PATCH] PYTHON-2992 Implement unified test format loop operation (#773) --- test/test_create_entities.py | 142 ++++++++++++++++++ ...client-storeEventsAsEntities-minItems.json | 18 +++ ...ity-client-storeEventsAsEntities-type.json | 18 +++ ...ailedEvent-hasServerConnectionId-type.json | 29 ++++ ...artedEvent-hasServerConnectionId-type.json | 29 ++++ ...eededEvent-hasServerConnectionId-type.json | 29 ++++ ...ntsAsEntities-conflict_with_client_id.json | 28 ++++ ...ities-conflict_within_different_array.json | 43 ++++++ ...AsEntities-conflict_within_same_array.json | 36 +++++ .../entity-findCursor-malformed.json | 44 ++++++ ...ind-cursor.json => entity-findCursor.json} | 12 +- .../ignoreResultAndError-malformed.json | 48 ++++++ .../valid-fail/ignoreResultAndError.json | 13 -- .../entity-client-storeEventsAsEntities.json | 67 +++++++++ test/unified_format.py | 95 ++++++++++-- 15 files changed, 613 insertions(+), 38 deletions(-) create mode 100644 test/test_create_entities.py create mode 100644 test/unified-test-format/invalid/entity-client-storeEventsAsEntities-minItems.json create mode 100644 test/unified-test-format/invalid/entity-client-storeEventsAsEntities-type.json create mode 100644 test/unified-test-format/invalid/expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type.json create mode 100644 test/unified-test-format/invalid/expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type.json create mode 100644 test/unified-test-format/invalid/expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type.json create mode 100644 test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_with_client_id.json create mode 100644 test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_different_array.json create mode 100644 test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_same_array.json create mode 100644 test/unified-test-format/valid-fail/entity-findCursor-malformed.json rename test/unified-test-format/valid-fail/{entity-find-cursor.json => entity-findCursor.json} (76%) create mode 100644 test/unified-test-format/valid-fail/ignoreResultAndError-malformed.json create mode 100644 test/unified-test-format/valid-pass/entity-client-storeEventsAsEntities.json diff --git a/test/test_create_entities.py b/test/test_create_entities.py new file mode 100644 index 000000000..9b5c30d64 --- /dev/null +++ b/test/test_create_entities.py @@ -0,0 +1,142 @@ +# Copyright 2021-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from test.unified_format import UnifiedSpecTestMixinV1 + +from pymongo.monitoring import PoolCreatedEvent + + +class TestCreateEntities(unittest.TestCase): + def test_store_events_as_entities(self): + self.scenario_runner = UnifiedSpecTestMixinV1() + spec = { + "description": "blank", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "events1", + "events": [ + "PoolCreatedEvent", + ] + } + ] + } + }, + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] + } + self.scenario_runner.TEST_SPEC = spec + self.scenario_runner.setUp() + self.scenario_runner.run_scenario(spec["tests"][0]) + final_entity_map = self.scenario_runner.entity_map + self.assertIn("events1", final_entity_map) + self.assertGreater(len(final_entity_map["events1"]), 0) + for event in final_entity_map["events1"]: + self.assertEqual(type(event), PoolCreatedEvent) + + def test_store_all_others_as_entities(self): + self.scenario_runner = UnifiedSpecTestMixinV1() + spec = { + "description": "Find", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "uriOptions": { + "retryReads": True + }, + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "dat" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "dat" + } + } + ], + + "tests": [ + { + "description": "test loops", + "operations": [ + { + "name": "loop", + "object": "testRunner", + "arguments": { + "storeIterationsAsEntity": "iterations", + "storeSuccessesAsEntity": "successes", + "storeFailuresAsEntity": "failures", + "storeErrorsAsEntity": "errors", + "numIterations": 5, + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "x": 44 + } + } + + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "x": 44 + } + } + + } + ] + } + } + ] + } + ] + } + + self.scenario_runner.TEST_SPEC = spec + self.scenario_runner.setUp() + self.scenario_runner.run_scenario(spec["tests"][0]) + final_entity_map = self.scenario_runner.entity_map + for entity in ["errors", "failures"]: + self.assertIn(entity, final_entity_map) + self.assertGreaterEqual(len(final_entity_map[entity]), 0) + self.assertEqual(type(final_entity_map[entity]), list) + for entity in ["successes", "iterations"]: + self.assertIn(entity, final_entity_map) + self.assertEqual(type(final_entity_map[entity]), int) diff --git a/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-minItems.json b/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-minItems.json new file mode 100644 index 000000000..d94863ed1 --- /dev/null +++ b/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-minItems.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-storeEventsAsEntities-minItems", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-type.json b/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-type.json new file mode 100644 index 000000000..79f6b85ed --- /dev/null +++ b/test/unified-test-format/invalid/entity-client-storeEventsAsEntities-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-storeEventsAsEntities-type", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/invalid/expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type.json b/test/unified-test-format/invalid/expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type.json new file mode 100644 index 000000000..7787ea651 --- /dev/null +++ b/test/unified-test-format/invalid/expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type.json @@ -0,0 +1,29 @@ +{ + "description": "expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type", + "schemaVersion": "1.6", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandFailedEvent": { + "hasServerConnectionId": "foo" + } + } + ] + } + ] + } + ] +} diff --git a/test/unified-test-format/invalid/expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type.json b/test/unified-test-format/invalid/expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type.json new file mode 100644 index 000000000..a913f00ab --- /dev/null +++ b/test/unified-test-format/invalid/expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type.json @@ -0,0 +1,29 @@ +{ + "description": "expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type", + "schemaVersion": "1.6", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "hasServerConnectionId": "foo" + } + } + ] + } + ] + } + ] +} diff --git a/test/unified-test-format/invalid/expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type.json b/test/unified-test-format/invalid/expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type.json new file mode 100644 index 000000000..0712c3369 --- /dev/null +++ b/test/unified-test-format/invalid/expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type.json @@ -0,0 +1,29 @@ +{ + "description": "expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type", + "schemaVersion": "1.6", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandSucceededEvent": { + "hasServerConnectionId": "foo" + } + } + ] + } + ] + } + ] +} diff --git a/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_with_client_id.json b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_with_client_id.json new file mode 100644 index 000000000..8c0c4d204 --- /dev/null +++ b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_with_client_id.json @@ -0,0 +1,28 @@ +{ + "description": "entity-client-storeEventsAsEntities-conflict_with_client_id", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "client0", + "events": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent" + ] + } + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_different_array.json b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_different_array.json new file mode 100644 index 000000000..77bc4abf2 --- /dev/null +++ b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_different_array.json @@ -0,0 +1,43 @@ +{ + "description": "entity-client-storeEventsAsEntities-conflict_within_different_array", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "events", + "events": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent" + ] + } + ] + } + }, + { + "client": { + "id": "client1", + "storeEventsAsEntities": [ + { + "id": "events", + "events": [ + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent" + ] + } + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_same_array.json b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_same_array.json new file mode 100644 index 000000000..e1a949988 --- /dev/null +++ b/test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_same_array.json @@ -0,0 +1,36 @@ +{ + "description": "entity-client-storeEventsAsEntities-conflict_within_same_array", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "events", + "events": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent" + ] + }, + { + "id": "events", + "events": [ + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent" + ] + } + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/test/unified-test-format/valid-fail/entity-findCursor-malformed.json b/test/unified-test-format/valid-fail/entity-findCursor-malformed.json new file mode 100644 index 000000000..0956efa4c --- /dev/null +++ b/test/unified-test-format/valid-fail/entity-findCursor-malformed.json @@ -0,0 +1,44 @@ +{ + "description": "entity-findCursor-malformed", + "schemaVersion": "1.3", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "database0Name" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "databaseName": "database0Name", + "collectionName": "coll0", + "documents": [] + } + ], + "tests": [ + { + "description": "createFindCursor fails if filter is not specified", + "operations": [ + { + "name": "createFindCursor", + "object": "collection0", + "saveResultAsEntity": "cursor0" + } + ] + } + ] +} diff --git a/test/unified-test-format/valid-fail/entity-find-cursor.json b/test/unified-test-format/valid-fail/entity-findCursor.json similarity index 76% rename from test/unified-test-format/valid-fail/entity-find-cursor.json rename to test/unified-test-format/valid-fail/entity-findCursor.json index f4c5bcdf4..389e448c0 100644 --- a/test/unified-test-format/valid-fail/entity-find-cursor.json +++ b/test/unified-test-format/valid-fail/entity-findCursor.json @@ -1,5 +1,5 @@ { - "description": "entity-find-cursor", + "description": "entity-findCursor", "schemaVersion": "1.3", "createEntities": [ { @@ -30,16 +30,6 @@ } ], "tests": [ - { - "description": "createFindCursor fails if filter is not specified", - "operations": [ - { - "name": "createFindCursor", - "object": "collection0", - "saveResultAsEntity": "cursor0" - } - ] - }, { "description": "iterateUntilDocumentOrError fails if it references a nonexistent entity", "operations": [ diff --git a/test/unified-test-format/valid-fail/ignoreResultAndError-malformed.json b/test/unified-test-format/valid-fail/ignoreResultAndError-malformed.json new file mode 100644 index 000000000..b64779c72 --- /dev/null +++ b/test/unified-test-format/valid-fail/ignoreResultAndError-malformed.json @@ -0,0 +1,48 @@ +{ + "description": "ignoreResultAndError-malformed", + "schemaVersion": "1.3", + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": true + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "database0Name" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "database0Name", + "documents": [] + } + ], + "tests": [ + { + "description": "malformed operation fails if ignoreResultAndError is true", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "foo": "bar" + }, + "ignoreResultAndError": true + } + ] + } + ] +} diff --git a/test/unified-test-format/valid-fail/ignoreResultAndError.json b/test/unified-test-format/valid-fail/ignoreResultAndError.json index 4457040b4..01b2421a9 100644 --- a/test/unified-test-format/valid-fail/ignoreResultAndError.json +++ b/test/unified-test-format/valid-fail/ignoreResultAndError.json @@ -54,19 +54,6 @@ "ignoreResultAndError": false } ] - }, - { - "description": "malformed operation fails if ignoreResultAndError is true", - "operations": [ - { - "name": "insertOne", - "object": "collection0", - "arguments": { - "foo": "bar" - }, - "ignoreResultAndError": true - } - ] } ] } diff --git a/test/unified-test-format/valid-pass/entity-client-storeEventsAsEntities.json b/test/unified-test-format/valid-pass/entity-client-storeEventsAsEntities.json new file mode 100644 index 000000000..e37e5a1ac --- /dev/null +++ b/test/unified-test-format/valid-pass/entity-client-storeEventsAsEntities.json @@ -0,0 +1,67 @@ +{ + "description": "entity-client-storeEventsAsEntities", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "client0_events", + "events": [ + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent" + ] + } + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "test", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "storeEventsAsEntities captures events", + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + } + ] +} diff --git a/test/unified_format.py b/test/unified_format.py index 0a2f0b996..4c705299e 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -16,13 +16,14 @@ https://github.com/mongodb/specifications/blob/master/source/unified-test-format/unified-test-format.rst """ - +import collections import copy import datetime import functools import os import re import sys +import time import types from collections import abc @@ -68,6 +69,13 @@ from test.utils import ( JSON_OPTS = json_util.JSONOptions(tz_aware=False) +IS_INTERRUPTED = False + + +def interrupt_loop(): + global IS_INTERRUPTED + IS_INTERRUPTED = True + def with_metaclass(meta, *bases): """Create a base class with a metaclass. @@ -188,7 +196,7 @@ class NonLazyCursor(object): class EventListenerUtil(CMAPListener, CommandListener): def __init__(self, observe_events, ignore_commands, - observe_sensitive_commands): + observe_sensitive_commands, store_events, entity_map): self._event_types = set(name.lower() for name in observe_events) if observe_sensitive_commands: self._observe_sensitive_commands = True @@ -197,6 +205,15 @@ class EventListenerUtil(CMAPListener, CommandListener): self._observe_sensitive_commands = False self._ignore_commands = _SENSITIVE_COMMANDS | set(ignore_commands) self._ignore_commands.add('configurefailpoint') + self._event_mapping = collections.defaultdict(list) + self.entity_map = entity_map + if store_events: + for i in store_events: + id = i["id"] + events = (i.lower() for i in i["events"]) + for i in events: + self._event_mapping[i].append(id) + self.entity_map[id] = [] super(EventListenerUtil, self).__init__() def get_events(self, event_type): @@ -205,8 +222,11 @@ class EventListenerUtil(CMAPListener, CommandListener): return [e for e in self.events if 'Command' not in type(e).__name__] def add_event(self, event): - if type(event).__name__.lower() in self._event_types: + event_name = type(event).__name__.lower() + if event_name in self._event_types: super(EventListenerUtil, self).add_event(event) + for id in self._event_mapping[event_name]: + self.entity_map[id].append(event) def _command_event(self, event): if event.command_name.lower() not in self._ignore_commands: @@ -241,6 +261,12 @@ class EntityMapUtil(object): self._session_lsids = {} self.test = test_class + def __contains__(self, item): + return item in self._entities + + def __len__(self): + return len(self._entities) + def __getitem__(self, item): try: return self._entities[item] @@ -271,13 +297,13 @@ class EntityMapUtil(object): ignore_commands = spec.get('ignoreCommandMonitoringEvents', []) observe_sensitive_commands = spec.get( 'observeSensitiveCommands', False) - # TODO: PYTHON-2511 support storeEventsAsEntities - if len(observe_events) or len(ignore_commands): - ignore_commands = [cmd.lower() for cmd in ignore_commands] - listener = EventListenerUtil( - observe_events, ignore_commands, observe_sensitive_commands) - self._listeners[spec['id']] = listener - kwargs['event_listeners'] = [listener] + ignore_commands = [cmd.lower() for cmd in ignore_commands] + listener = EventListenerUtil( + observe_events, ignore_commands, + observe_sensitive_commands, + spec.get("storeEventsAsEntities"), self) + self._listeners[spec['id']] = listener + kwargs['event_listeners'] = [listener] if spec.get('useMultipleMongoses'): if client_context.load_balancer or client_context.serverless: kwargs['h'] = client_context.MULTI_MONGOS_LB_URI @@ -1048,6 +1074,47 @@ class UnifiedSpecTestMixinV1(IntegrationTest): pool = get_pool(client) self.assertEqual(spec['connections'], pool.active_sockets) + def _testOperation_loop(self, spec): + failure_key = spec.get('storeFailuresAsEntity') + error_key = spec.get('storeErrorsAsEntity') + successes_key = spec.get('storeSuccessesAsEntity') + iteration_key = spec.get('storeIterationsAsEntity') + iteration_limiter_key = spec.get('numIterations') + if failure_key: + self.entity_map[failure_key] = [] + if error_key: + self.entity_map[error_key] = [] + if successes_key: + self.entity_map[successes_key] = 0 + if iteration_key: + self.entity_map[iteration_key] = 0 + i = 0 + while True: + if iteration_limiter_key and i >= iteration_limiter_key: + break + i += 1 + if IS_INTERRUPTED: + break + try: + for op in spec["operations"]: + self.run_entity_operation(op) + if successes_key: + self.entity_map._entities[successes_key] += 1 + if iteration_key: + self.entity_map._entities[iteration_key] += 1 + except AssertionError as exc: + if failure_key or error_key: + self.entity_map[failure_key or error_key].append({ + "error": exc, "time": time.time()}) + else: + raise exc + except Exception as exc: + if error_key or failure_key: + self.entity_map[error_key or failure_key].append( + {"error": exc, "time": time.time()}) + else: + raise exc + def run_special_operation(self, spec): opname = spec['name'] method_name = '_testOperation_%s' % (opname,) @@ -1060,11 +1127,11 @@ class UnifiedSpecTestMixinV1(IntegrationTest): def run_operations(self, spec): for op in spec: - target = op['object'] - if target != 'testRunner': - self.run_entity_operation(op) - else: + if op['object'] == 'testRunner': self.run_special_operation(op) + else: + self.run_entity_operation(op) + def check_events(self, spec): for event_spec in spec: