PYTHON-2951 Test that handshake errors are retryable (#846)

This commit is contained in:
Shane Harvey 2022-02-23 14:00:43 -08:00 committed by GitHub
parent 6fb8d7afe8
commit f8f34b0438
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
51 changed files with 985 additions and 3 deletions

View File

@ -0,0 +1,257 @@
{
"description": "retryable reads handshake failures",
"schemaVersion": "1.3",
"runOnRequirements": [
{
"minServerVersion": "4.2",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
],
"auth": true
}
],
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": false,
"observeEvents": [
"commandStartedEvent",
"connectionCheckOutStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-handshake-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll"
}
}
],
"initialData": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
},
{
"_id": 3,
"x": 33
}
]
}
],
"tests": [
{
"description": "find succeeds after retryable handshake network error",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "find",
"object": "collection0",
"arguments": {
"filter": {
"_id": 2
}
},
"expectResult": [
{
"_id": 2,
"x": 22
}
]
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"find": "coll",
"filter": {
"_id": 2
}
},
"databaseName": "retryable-handshake-tests"
}
}
]
}
]
},
{
"description": "find succeeds after retryable handshake network error (ShutdownInProgress)",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"errorCode": 91
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "find",
"object": "collection0",
"arguments": {
"filter": {
"_id": 2
}
},
"expectResult": [
{
"_id": 2,
"x": 22
}
]
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"find": "coll",
"filter": {
"_id": 2
}
},
"databaseName": "retryable-handshake-tests"
}
}
]
}
]
}
]
}

View File

@ -0,0 +1,279 @@
{
"description": "retryable writes handshake failures",
"schemaVersion": "1.3",
"runOnRequirements": [
{
"minServerVersion": "4.2",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
],
"auth": true
}
],
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": false,
"observeEvents": [
"commandStartedEvent",
"connectionCheckOutStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-handshake-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll"
}
}
],
"initialData": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
],
"tests": [
{
"description": "InsertOne succeeds after retryable handshake error",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"document": {
"_id": 2,
"x": 22
}
}
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"insert": "coll",
"documents": [
{
"_id": 2,
"x": 22
}
]
},
"commandName": "insert",
"databaseName": "retryable-handshake-tests"
}
}
]
}
],
"outcome": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
}
]
}
]
},
{
"description": "InsertOne succeeds after retryable handshake error ShutdownInProgress",
"operations": [
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"errorCode": 91
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
}
},
"expectError": {
"isError": true
}
},
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"document": {
"_id": 2,
"x": 22
}
}
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"insert": "coll",
"documents": [
{
"_id": 2,
"x": 22
}
]
},
"commandName": "insert",
"databaseName": "retryable-handshake-tests"
}
}
]
}
],
"outcome": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
}
]
}
]
}
]
}

View File

@ -46,7 +46,7 @@ from pymongo.monitoring import (
from pymongo.write_concern import WriteConcern
# Location of JSON test specifications.
_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_reads")
_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_reads", "legacy")
class TestClientOptions(PyMongoTestCase):

View File

@ -0,0 +1,32 @@
# Copyright 2022-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test the Retryable Reads unified spec tests."""
import os
import sys
sys.path[0:0] = [""]
from test import unittest
from test.unified_format import generate_test_classes
# Location of JSON test specifications.
TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "retryable_reads", "unified")
# Generate unified tests.
globals().update(generate_test_classes(TEST_PATH, module=__name__))
if __name__ == "__main__":
unittest.main()

View File

@ -0,0 +1,204 @@
{
"description": "retryable abortTransaction on handshake errors",
"schemaVersion": "1.4",
"runOnRequirements": [
{
"minServerVersion": "4.2",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
],
"serverless": "forbid",
"auth": true
}
],
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": false,
"observeEvents": [
"commandStartedEvent",
"connectionCheckOutStartedEvent"
]
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-handshake-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll"
}
},
{
"session": {
"id": "session0",
"client": "client0"
}
},
{
"session": {
"id": "session1",
"client": "client0"
}
}
],
"initialData": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
],
"tests": [
{
"description": "AbortTransaction succeeds after handshake network error",
"skipReason": "DRIVERS-2032: Pinned servers need to be checked if they are still selectable",
"operations": [
{
"name": "startTransaction",
"object": "session0"
},
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"session": "session0",
"document": {
"_id": 2,
"x": 22
}
}
},
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"session": "session1",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
},
"session": "session1"
},
"expectError": {
"isError": true
}
},
{
"name": "abortTransaction",
"object": "session0"
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"insert": "coll",
"documents": [
{
"_id": 2,
"x": 22
}
],
"startTransaction": true
},
"commandName": "insert",
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"abortTransaction": 1,
"lsid": {
"$$sessionLsid": "session0"
}
},
"commandName": "abortTransaction",
"databaseName": "admin"
}
}
]
}
],
"outcome": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
]
}
]
}

View File

@ -0,0 +1,211 @@
{
"description": "retryable commitTransaction on handshake errors",
"schemaVersion": "1.4",
"runOnRequirements": [
{
"minServerVersion": "4.2",
"topologies": [
"replicaset",
"sharded",
"load-balanced"
],
"serverless": "forbid",
"auth": true
}
],
"createEntities": [
{
"client": {
"id": "client0",
"useMultipleMongoses": false,
"observeEvents": [
"commandStartedEvent",
"connectionCheckOutStartedEvent"
],
"uriOptions": {
"retryWrites": false
}
}
},
{
"database": {
"id": "database0",
"client": "client0",
"databaseName": "retryable-handshake-tests"
}
},
{
"collection": {
"id": "collection0",
"database": "database0",
"collectionName": "coll"
}
},
{
"session": {
"id": "session0",
"client": "client0"
}
},
{
"session": {
"id": "session1",
"client": "client0"
}
}
],
"initialData": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
}
]
}
],
"tests": [
{
"description": "CommitTransaction succeeds after handshake network error",
"skipReason": "DRIVERS-2032: Pinned servers need to be checked if they are still selectable",
"operations": [
{
"name": "startTransaction",
"object": "session0"
},
{
"name": "insertOne",
"object": "collection0",
"arguments": {
"session": "session0",
"document": {
"_id": 2,
"x": 22
}
}
},
{
"name": "failPoint",
"object": "testRunner",
"arguments": {
"client": "client0",
"session": "session1",
"failPoint": {
"configureFailPoint": "failCommand",
"mode": {
"times": 2
},
"data": {
"failCommands": [
"saslContinue",
"ping"
],
"closeConnection": true
}
}
}
},
{
"name": "runCommand",
"object": "database0",
"arguments": {
"commandName": "ping",
"command": {
"ping": 1
},
"session": "session1"
},
"expectError": {
"isError": true
}
},
{
"name": "commitTransaction",
"object": "session0"
}
],
"expectEvents": [
{
"client": "client0",
"eventType": "cmap",
"events": [
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
},
{
"connectionCheckOutStartedEvent": {}
}
]
},
{
"client": "client0",
"events": [
{
"commandStartedEvent": {
"command": {
"insert": "coll",
"documents": [
{
"_id": 2,
"x": 22
}
],
"startTransaction": true
},
"commandName": "insert",
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"ping": 1
},
"databaseName": "retryable-handshake-tests"
}
},
{
"commandStartedEvent": {
"command": {
"commitTransaction": 1,
"lsid": {
"$$sessionLsid": "session0"
}
},
"commandName": "commitTransaction",
"databaseName": "admin"
}
}
]
}
],
"outcome": [
{
"collectionName": "coll",
"databaseName": "retryable-handshake-tests",
"documents": [
{
"_id": 1,
"x": 11
},
{
"_id": 2,
"x": 22
}
]
}
]
}
]
}

View File

@ -1153,8 +1153,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest):
self.assertEqual(actual_events, [])
continue
if len(events) > len(actual_events):
self.fail("Expected to see %s events, got %s" % (len(events), len(actual_events)))
self.assertGreaterEqual(len(actual_events), len(events), actual_events)
for idx, expected_event in enumerate(events):
self.match_evaluator.match_event(event_type, expected_event, actual_events[idx])