mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PYTHON-2992 Implement unified test format loop operation (#773)

Browse Source
This commit is contained in:
Julius Park 2021-11-08 16:19:24 -08:00 committed by GitHub
parent b05ac0e7ba
commit e80141ed1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 613 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
test/test_create_entities.py Normal file
View File

@ -0,0 +1,142 @@
# Copyright 2021-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from test.unified_format import UnifiedSpecTestMixinV1
from pymongo.monitoring import PoolCreatedEvent
class TestCreateEntities(unittest.TestCase):
def test_store_events_as_entities(self):
self.scenario_runner = UnifiedSpecTestMixinV1()
spec = {
"description": "blank",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": [
{
"id": "events1",
"events": [
"PoolCreatedEvent",
]
}
]
}
},
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}
self.scenario_runner.TEST_SPEC = spec
self.scenario_runner.setUp()
self.scenario_runner.run_scenario(spec["tests"][0])
final_entity_map = self.scenario_runner.entity_map
self.assertIn("events1", final_entity_map)
self.assertGreater(len(final_entity_map["events1"]), 0)
for event in final_entity_map["events1"]:
self.assertEqual(type(event), PoolCreatedEvent)
def test_store_all_others_as_entities(self):
self.scenario_runner = UnifiedSpecTestMixinV1()
spec = {
"description": "Find",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"uriOptions": {
"retryReads": True
},
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "dat"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "dat"
}
}
],
"tests": [
{
"description": "test loops",
"operations": [
{
"name": "loop",
"object": "testRunner",
"arguments": {
"storeIterationsAsEntity": "iterations",
"storeSuccessesAsEntity": "successes",
"storeFailuresAsEntity": "failures",
"storeErrorsAsEntity": "errors",
"numIterations": 5,
"operations": [
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"document": {
"_id": 1,
"x": 44
}
}
},
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"document": {
"_id": 1,
"x": 44
}
}
}
]
}
}
]
}
]
}
self.scenario_runner.TEST_SPEC = spec
self.scenario_runner.setUp()
self.scenario_runner.run_scenario(spec["tests"][0])
final_entity_map = self.scenario_runner.entity_map
for entity in ["errors", "failures"]:
self.assertIn(entity, final_entity_map)
self.assertGreaterEqual(len(final_entity_map[entity]), 0)
self.assertEqual(type(final_entity_map[entity]), list)
for entity in ["successes", "iterations"]:
self.assertIn(entity, final_entity_map)
self.assertEqual(type(final_entity_map[entity]), int)

18
test/unified-test-format/invalid/entity-client-storeEventsAsEntities-minItems.json Normal file
View File

@ -0,0 +1,18 @@
{
"description": "entity-client-storeEventsAsEntities-minItems",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": []
}
}
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}

18
test/unified-test-format/invalid/entity-client-storeEventsAsEntities-type.json Normal file
View File

@ -0,0 +1,18 @@
{
"description": "entity-client-storeEventsAsEntities-type",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": 0
}
}
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}

29
test/unified-test-format/invalid/expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type.json Normal file
View File

@ -0,0 +1,29 @@
{
"description": "expectedCommandEvent-commandFailedEvent-hasServerConnectionId-type",
"schemaVersion": "1.6",
"createEntities": [
{
"client": {
"id": "client0"
}
}
],
"tests": [
{
"description": "foo",
"operations": [],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandFailedEvent": {
"hasServerConnectionId": "foo"
}
}
]
}
]
}
]
}

29
test/unified-test-format/invalid/expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type.json Normal file
View File

@ -0,0 +1,29 @@
{
"description": "expectedCommandEvent-commandStartedEvent-hasServerConnectionId-type",
"schemaVersion": "1.6",
"createEntities": [
{
"client": {
"id": "client0"
}
}
],
"tests": [
{
"description": "foo",
"operations": [],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"hasServerConnectionId": "foo"
}
}
]
}
]
}
]
}

29
test/unified-test-format/invalid/expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type.json Normal file
View File

@ -0,0 +1,29 @@
{
"description": "expectedCommandEvent-commandSucceededEvent-hasServerConnectionId-type",
"schemaVersion": "1.6",
"createEntities": [
{
"client": {
"id": "client0"
}
}
],
"tests": [
{
"description": "foo",
"operations": [],
"expectEvents": [
{
"client": "client0",
"events": [
{
"commandSucceededEvent": {
"hasServerConnectionId": "foo"
}
}
]
}
]
}
]
}

28
test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_with_client_id.json Normal file
View File

@ -0,0 +1,28 @@
{
"description": "entity-client-storeEventsAsEntities-conflict_with_client_id",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": [
{
"id": "client0",
"events": [
"PoolCreatedEvent",
"PoolReadyEvent",
"PoolClearedEvent",
"PoolClosedEvent"
]
}
]
}
}
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}

43
test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_different_array.json Normal file
View File

@ -0,0 +1,43 @@
{
"description": "entity-client-storeEventsAsEntities-conflict_within_different_array",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": [
{
"id": "events",
"events": [
"PoolCreatedEvent",
"PoolReadyEvent",
"PoolClearedEvent",
"PoolClosedEvent"
]
}
]
}
},
{
"client": {
"id": "client1",
"storeEventsAsEntities": [
{
"id": "events",
"events": [
"CommandStartedEvent",
"CommandSucceededEvent",
"CommandFailedEvent"
]
}
]
}
}
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}

36
test/unified-test-format/valid-fail/entity-client-storeEventsAsEntities-conflict_within_same_array.json Normal file
View File

@ -0,0 +1,36 @@
{
"description": "entity-client-storeEventsAsEntities-conflict_within_same_array",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": [
{
"id": "events",
"events": [
"PoolCreatedEvent",
"PoolReadyEvent",
"PoolClearedEvent",
"PoolClosedEvent"
]
},
{
"id": "events",
"events": [
"CommandStartedEvent",
"CommandSucceededEvent",
"CommandFailedEvent"
]
}
]
}
}
],
"tests": [
{
"description": "foo",
"operations": []
}
]
}

44
test/unified-test-format/valid-fail/entity-findCursor-malformed.json Normal file
View File

@ -0,0 +1,44 @@
{
"description": "entity-findCursor-malformed",
"schemaVersion": "1.3",
"createEntities": [
{
"client": {
"id": "client0"
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "database0Name"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"databaseName": "database0Name",
"collectionName": "coll0",
"documents": []
}
],
"tests": [
{
"description": "createFindCursor fails if filter is not specified",
"operations": [
{
"name": "createFindCursor",
"object": "collection0",
"saveResultAsEntity": "cursor0"
}
]
}
]
}

12
test/unified-test-format/valid-fail/entity-find-cursor.json → test/unified-test-format/valid-fail/entity-findCursor.json
View File

@ -1,5 +1,5 @@
{
"description": "entity-find-cursor",
"description": "entity-findCursor",
"schemaVersion": "1.3",
"createEntities": [
{
@ -30,16 +30,6 @@
}
],
"tests": [
{
"description": "createFindCursor fails if filter is not specified",
"operations": [
{
"name": "createFindCursor",
"object": "collection0",
"saveResultAsEntity": "cursor0"
}
]
},
{
"description": "iterateUntilDocumentOrError fails if it references a nonexistent entity",
"operations": [

48
test/unified-test-format/valid-fail/ignoreResultAndError-malformed.json Normal file
View File

@ -0,0 +1,48 @@
{
"description": "ignoreResultAndError-malformed",
"schemaVersion": "1.3",
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": true
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "database0Name"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "database0Name",
"documents": []
}
],
"tests": [
{
"description": "malformed operation fails if ignoreResultAndError is true",
"operations": [
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"foo": "bar"
},
"ignoreResultAndError": true
}
]
}
]
}

13
test/unified-test-format/valid-fail/ignoreResultAndError.json
View File

@ -54,19 +54,6 @@
"ignoreResultAndError": false
}
]
},
{
"description": "malformed operation fails if ignoreResultAndError is true",
"operations": [
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"foo": "bar"
},
"ignoreResultAndError": true
}
]
}
]
}

67
test/unified-test-format/valid-pass/entity-client-storeEventsAsEntities.json Normal file
View File

@ -0,0 +1,67 @@
{
"description": "entity-client-storeEventsAsEntities",
"schemaVersion": "1.2",
"createEntities": [
{
"client": {
"id": "client0",
"storeEventsAsEntities": [
{
"id": "client0_events",
"events": [
"CommandStartedEvent",
"CommandSucceededEvent",
"CommandFailedEvent"
]
}
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "test"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll0"
}
}
],
"initialData": [
{
"collectionName": "coll0",
"databaseName": "test",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
],
"tests": [
{
"description": "storeEventsAsEntities captures events",
"operations": [
{
"name": "find",
"object": "collection0",
"arguments": {
"filter": {}
},
"expectResult": [
{
"_id": 1,
"x": 11
}
]
}
]
}
]
}

95
test/unified_format.py
View File

@ -16,13 +16,14 @@
https://github.com/mongodb/specifications/blob/master/source/unified-test-format/unified-test-format.rst
"""
import collections
import copy
import datetime
import functools
import os
import re
import sys
import time
import types
from collections import abc
@ -68,6 +69,13 @@ from test.utils import (
JSON_OPTS = json_util.JSONOptions(tz_aware=False)
IS_INTERRUPTED = False
def interrupt_loop():
global IS_INTERRUPTED
IS_INTERRUPTED = True
def with_metaclass(meta, *bases):
"""Create a base class with a metaclass.
@ -188,7 +196,7 @@ class NonLazyCursor(object):
class EventListenerUtil(CMAPListener, CommandListener):
def __init__(self, observe_events, ignore_commands,
observe_sensitive_commands):
observe_sensitive_commands, store_events, entity_map):
self._event_types = set(name.lower() for name in observe_events)
if observe_sensitive_commands:
self._observe_sensitive_commands = True
@ -197,6 +205,15 @@ class EventListenerUtil(CMAPListener, CommandListener):
self._observe_sensitive_commands = False
self._ignore_commands = _SENSITIVE_COMMANDS | set(ignore_commands)
self._ignore_commands.add('configurefailpoint')
self._event_mapping = collections.defaultdict(list)
self.entity_map = entity_map
if store_events:
for i in store_events:
id = i["id"]
events = (i.lower() for i in i["events"])
for i in events:
self._event_mapping[i].append(id)
self.entity_map[id] = []
super(EventListenerUtil, self).__init__()
def get_events(self, event_type):
@ -205,8 +222,11 @@ class EventListenerUtil(CMAPListener, CommandListener):
return [e for e in self.events if 'Command' not in type(e).__name__]
def add_event(self, event):
if type(event).__name__.lower() in self._event_types:
event_name = type(event).__name__.lower()
if event_name in self._event_types:
super(EventListenerUtil, self).add_event(event)
for id in self._event_mapping[event_name]:
self.entity_map[id].append(event)
def _command_event(self, event):
if event.command_name.lower() not in self._ignore_commands:
@ -241,6 +261,12 @@ class EntityMapUtil(object):
self._session_lsids = {}
self.test = test_class
def __contains__(self, item):
return item in self._entities
def __len__(self):
return len(self._entities)
def __getitem__(self, item):
try:
return self._entities[item]
@ -271,13 +297,13 @@ class EntityMapUtil(object):
ignore_commands = spec.get('ignoreCommandMonitoringEvents', [])
observe_sensitive_commands = spec.get(
'observeSensitiveCommands', False)
# TODO: PYTHON-2511 support storeEventsAsEntities
if len(observe_events) or len(ignore_commands):
ignore_commands = [cmd.lower() for cmd in ignore_commands]
listener = EventListenerUtil(
observe_events, ignore_commands, observe_sensitive_commands)
self._listeners[spec['id']] = listener
kwargs['event_listeners'] = [listener]
ignore_commands = [cmd.lower() for cmd in ignore_commands]
listener = EventListenerUtil(
observe_events, ignore_commands,
observe_sensitive_commands,
spec.get("storeEventsAsEntities"), self)
self._listeners[spec['id']] = listener
kwargs['event_listeners'] = [listener]
if spec.get('useMultipleMongoses'):
if client_context.load_balancer or client_context.serverless:
kwargs['h'] = client_context.MULTI_MONGOS_LB_URI
@ -1048,6 +1074,47 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
pool = get_pool(client)
self.assertEqual(spec['connections'], pool.active_sockets)
def _testOperation_loop(self, spec):
failure_key = spec.get('storeFailuresAsEntity')
error_key = spec.get('storeErrorsAsEntity')
successes_key = spec.get('storeSuccessesAsEntity')
iteration_key = spec.get('storeIterationsAsEntity')
iteration_limiter_key = spec.get('numIterations')
if failure_key:
self.entity_map[failure_key] = []
if error_key:
self.entity_map[error_key] = []
if successes_key:
self.entity_map[successes_key] = 0
if iteration_key:
self.entity_map[iteration_key] = 0
i = 0
while True:
if iteration_limiter_key and i >= iteration_limiter_key:
break
i += 1
if IS_INTERRUPTED:
break
try:
for op in spec["operations"]:
self.run_entity_operation(op)
if successes_key:
self.entity_map._entities[successes_key] += 1
if iteration_key:
self.entity_map._entities[iteration_key] += 1
except AssertionError as exc:
if failure_key or error_key:
self.entity_map[failure_key or error_key].append({
"error": exc, "time": time.time()})
else:
raise exc
except Exception as exc:
if error_key or failure_key:
self.entity_map[error_key or failure_key].append(
{"error": exc, "time": time.time()})
else:
raise exc
def run_special_operation(self, spec):
opname = spec['name']
method_name = '_testOperation_%s' % (opname,)
@ -1060,11 +1127,11 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
def run_operations(self, spec):
for op in spec:
target = op['object']
if target != 'testRunner':
self.run_entity_operation(op)
else:
if op['object'] == 'testRunner':
self.run_special_operation(op)
else:
self.run_entity_operation(op)
def check_events(self, spec):
for event_spec in spec: