PYTHON-694 Test mod_wsgi sub interpreters (#1327)

Test mod_wsgi sub interpreters and embedded mode.
Use unique collection name for each mod_wsgi interpreter.
Test encoding/decoding all bson types.

.evergreen/config.yml

@@ -347,7 +347,9 @@ functions:
         script: |
           set -o xtrace
           ${PREPARE_SHELL}
-          PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh
+          PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} \
+            MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
+            bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh
 
   "run mockupdb tests":
     - command: shell.exec
@@ -1677,6 +1679,28 @@ tasks:
             TOPOLOGY: "replica_set"
         - func: "run mod_wsgi tests"
 
+    - name: "mod-wsgi-embedded-mode-standalone"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "server"
+        - func: "run mod_wsgi tests"
+          vars:
+            MOD_WSGI_EMBEDDED: "1"
+
+    - name: "mod-wsgi-embedded-mode-replica-set"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "replica_set"
+        - func: "run mod_wsgi tests"
+          vars:
+            MOD_WSGI_EMBEDDED: "1"
+
     - name: "no-server"
       tags: ["no-server"]
       commands:
@@ -3088,6 +3112,8 @@ buildvariants:
   tasks:
      - name: "mod-wsgi-standalone"
      - name: "mod-wsgi-replica-set"
+     - name: "mod-wsgi-embedded-mode-standalone"
+     - name: "mod-wsgi-embedded-mode-replica-set"
 
 - matrix_name: "mockupdb-tests"
   matrix_spec:

.evergreen/run-mod-wsgi-tests.sh

@@ -18,25 +18,30 @@ fi
 
 PYTHON_VERSION=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('.'.join(str(val) for val in sys.version_info[:2]))")
 
+# Ensure the C extensions are installed.
+${PYTHON_BINARY} setup.py build_ext -i
+
 export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_version/$MOD_WSGI_VERSION/mod_wsgi.so
 export PYTHONHOME=/opt/python/$PYTHON_VERSION
+# If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead
+# of daemon mode (WSGIDaemonProcess).
+if [ -n "$MOD_WSGI_EMBEDDED" ]; then
+    export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf
+else
+    export MOD_WSGI_CONF=mod_wsgi_test.conf
+fi
 
 cd ..
 $APACHE -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG}
 trap '$APACHE -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG}' EXIT HUP
 
-set +e
-wget -t 1 -T 10 -O - "http://localhost:8080${PROJECT_DIRECTORY}"
-STATUS=$?
-set -e
+wget -t 1 -T 10 -O - "http://localhost:8080/interpreter1${PROJECT_DIRECTORY}" || (cat error_log && exit 1)
+wget -t 1 -T 10 -O - "http://localhost:8080/interpreter2${PROJECT_DIRECTORY}" || (cat error_log && exit 1)
 
-# Debug
-cat error_log
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel \
+    http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \
+    (tail -n 100 error_log && exit 1)
 
-if [ $STATUS != 0 ]; then
-    exit $STATUS
-fi
-
-${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY}
-
-${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY}
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial \
+    http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \
+    (tail -n 100 error_log && exit 1)

test/mod_wsgi_test/README.rst

@@ -15,7 +15,7 @@ Test Matrix
 
 PyMongo should be tested with several versions of mod_wsgi and a selection
 of Python versions. Each combination of mod_wsgi and Python version should
-be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi``
+be tested with a standalone and a replica set. ``mod_wsgi_test.py``
 detects if the deployment is a replica set and connects to the whole set.
 
 Setup
@@ -74,31 +74,37 @@ Run the test
 Run the included ``test_client.py`` script::
 
     python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \
-         http://localhost/${WORKSPACE}
+         http://localhost/interpreter1${WORKSPACE} http://localhost/interpreter2${WORKSPACE}
 
 ...where the "n" argument is the total number of requests to make to Apache,
 and "t" specifies the number of threads. ``WORKSPACE`` is the location of
-the PyMongo checkout.
+the PyMongo checkout. Note that multiple URLs are passed, each one corresponds
+to a different sub interpreter.
 
 Run this script again with different arguments to make serial requests::
 
     python test/mod_wsgi_test/test_client.py -n 25000 serial \
-        http://localhost/${WORKSPACE}
+        http://localhost/interpreter1${WORKSPACE} http://localhost/interpreter2${WORKSPACE}
 
 The ``test_client.py`` script merely makes HTTP requests to Apache. Its
 exit code is non-zero if any of its requests fails, for example with an
 HTTP 500.
 
-The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi``.
+The core of the test is in the WSGI script, ``mod_wsgi_test.py``.
 This script inserts some documents into MongoDB at startup, then queries
 documents for each HTTP request.
 
 If PyMongo is leaking connections and "n" is much greater than the ulimit,
 the test will fail when PyMongo exhausts its file descriptors.
 
+The script also encodes and decodes all BSON types to ensure that
+multiple sub interpreters in the same process are supported. This tests
+the workaround added in `PYTHON-569 <https://jira.mongodb.org/browse/PYTHON-569>`_.
+
 Automation
 ----------
 
 At MongoDB, Inc. we use a continuous integration job that tests each
 combination in the matrix. The job starts up Apache, starts a single server
 or replica set, and runs ``test_client.py`` with the proper arguments.
+See `run-mod-wsgi-tests.sh <https://github.com/mongodb/mongo-python-driver/blob/master/.evergreen/run-mod-wsgi-tests.sh>`_

test/mod_wsgi_test/apache22amazon.conf

@@ -31,4 +31,4 @@ CustomLog ${PWD}/access_log combined
     Allow from All
 </Directory>
 
-Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF}

test/mod_wsgi_test/apache22ubuntu1204.conf

@@ -26,4 +26,4 @@ CustomLog ${PWD}/access_log combined
     Allow from All
 </Directory>
 
-Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF}

test/mod_wsgi_test/apache24ubuntu161404.conf

@@ -25,4 +25,4 @@ CustomLog ${PWD}/access_log combined
     Require all granted
 </Directory>
 
-Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF}

test/mod_wsgi_test/mod_wsgi_test.conf

@@ -1,4 +1,4 @@
-# Copyright 2012-2015 MongoDB, Inc.
+# Copyright 2012-present MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,17 +20,13 @@ LoadModule wsgi_module ${MOD_WSGI_SO}
 WSGISocketPrefix /tmp/
 
 <VirtualHost *>
-
     ServerName localhost
-
     WSGIDaemonProcess mod_wsgi_test processes=1 threads=15 display-name=mod_wsgi_test
-
     WSGIProcessGroup mod_wsgi_test
-
+    # Mount the script twice so that multiple interpreters are used.
     # For the convenience of unittests, rather than hard-code the location of
-    # mod_wsgi_test.wsgi, include it in the URL, so
-    # http://localhost/location-of-pymongo-checkout will work:
-
-    WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi
-
+    # mod_wsgi_test.py, include it in the URL, so
+    # http://localhost/interpreter1/location-of-pymongo-checkout will work:
+    WSGIScriptAliasMatch ^/interpreter1/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py
+    WSGIScriptAliasMatch ^/interpreter2/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py
 </VirtualHost>

test/mod_wsgi_test/mod_wsgi_test.py

@@ -0,0 +1,110 @@
+# Copyright 2012-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Minimal test of PyMongo in a WSGI application, see bug PYTHON-353
+"""
+
+import datetime
+import os
+import re
+import sys
+import uuid
+
+this_path = os.path.dirname(os.path.join(os.getcwd(), __file__))
+
+# Location of PyMongo checkout
+repository_path = os.path.normpath(os.path.join(this_path, "..", ".."))
+sys.path.insert(0, repository_path)
+
+import bson
+import pymongo
+from bson.binary import STANDARD, Binary
+from bson.code import Code
+from bson.codec_options import CodecOptions
+from bson.datetime_ms import DatetimeConversion, DatetimeMS
+from bson.dbref import DBRef
+from bson.objectid import ObjectId
+from bson.regex import Regex
+from pymongo.mongo_client import MongoClient
+
+# Ensure the C extensions are installed.
+assert bson.has_c()
+assert pymongo.has_c()
+
+OPTS: "CodecOptions[dict]" = CodecOptions(
+    uuid_representation=STANDARD, datetime_conversion=DatetimeConversion.DATETIME_AUTO
+)
+client: "MongoClient[dict]" = MongoClient()
+# Use a unique collection name for each process:
+coll_name = f"test-{uuid.uuid4()}"
+collection = client.test.get_collection(coll_name, codec_options=OPTS)
+ndocs = 20
+collection.drop()
+doc = {
+    "int32": 2 << 15,
+    "int64": 2 << 50,
+    "null": None,
+    "bool": True,
+    "float": 1.5,
+    "str": "string",
+    "list": [1, 2, 3],
+    "dict": {"a": 1, "b": 2, "c": 3},
+    "datetime": datetime.datetime.fromtimestamp(1690328577.446),
+    "datetime_ms_out_of_range": DatetimeMS(-2 << 60),
+    "regex_native": re.compile("regex*"),
+    "regex_pymongo": Regex("regex*"),
+    "binary": Binary(b"bytes", 128),
+    "oid": ObjectId(),
+    "dbref": DBRef("test", 1),
+    "code": Code("function(){ return true; }"),
+    "code_w_scope": Code("return function(){ return x; }", scope={"x": False}),
+    "bytes": b"bytes",
+    "uuid": uuid.uuid4(),
+}
+collection.insert_many([dict(i=i, **doc) for i in range(ndocs)])
+client.close()  # Discard main thread's request socket.
+client = MongoClient()
+collection = client.test.get_collection(coll_name, codec_options=OPTS)
+
+try:
+    from mod_wsgi import version as mod_wsgi_version  # type: ignore[import]
+except:
+    mod_wsgi_version = None
+
+
+def application(environ, start_response):
+    results = list(collection.find().batch_size(10))
+    assert len(results) == ndocs, f"n_actual={len(results)} n_expected={ndocs}"
+    # Test encoding and decoding works (for sub interpreter support).
+    decoded = bson.decode(bson.encode(doc, codec_options=OPTS), codec_options=OPTS)
+    for key, value in doc.items():
+        # Native regex objects are decoded as bson Regex.
+        if isinstance(value, re.Pattern):
+            value = Regex.from_native(value)
+        assert decoded[key] == value, f"failed on doc[{key!r}]: {decoded[key]!r} != {value!r}"
+        assert isinstance(
+            decoded[key], type(value)
+        ), f"failed on doc[{key}]: {decoded[key]!r} is not an instance of {type(value)}"
+
+    output = (
+        f" python {sys.version}, mod_wsgi {mod_wsgi_version},"
+        f" pymongo {pymongo.version},"
+        f' mod_wsgi.process_group = {environ["mod_wsgi.process_group"]!r}'
+        f' mod_wsgi.application_group = {environ["mod_wsgi.application_group"]!r}'
+        f' wsgi.multithread = {environ["wsgi.multithread"]!r}'
+        "\n"
+    )
+    response_headers = [("Content-Length", str(len(output)))]
+    start_response("200 OK", response_headers)
+    return [output.encode("ascii")]

test/mod_wsgi_test/mod_wsgi_test.wsgi

@@ -1,53 +0,0 @@
-# Copyright 2012-2015 MongoDB, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Minimal test of PyMongo in a WSGI application, see bug PYTHON-353
-"""
-
-import os
-import sys
-
-this_path = os.path.dirname(os.path.join(os.getcwd(), __file__))
-
-# Location of PyMongo checkout
-repository_path = os.path.normpath(os.path.join(this_path, '..', '..'))
-sys.path.insert(0, repository_path)
-
-import pymongo
-from pymongo.hello import HelloCompat  # noqa
-from pymongo.mongo_client import MongoClient
-
-client = MongoClient()
-collection = client.test.test
-ndocs = 20
-collection.drop()
-collection.insert_many([{'i': i} for i in range(ndocs)])
-client.close()  # Discard main thread's request socket.
-client = MongoClient()
-collection = client.test.test
-
-try:
-    from mod_wsgi import version as mod_wsgi_version
-except:
-    mod_wsgi_version = None
-
-
-def application(environ, start_response):
-    results = list(collection.find().batch_size(10))
-    assert len(results) == ndocs
-    output = ' python %s, mod_wsgi %s, pymongo %s ' % (
-        sys.version, mod_wsgi_version, pymongo.version)
-    response_headers = [('Content-Length', str(len(output)))]
-    start_response('200 OK', response_headers)
-    return [output.encode('ascii')]

test/mod_wsgi_test/mod_wsgi_test_embedded.conf

@@ -0,0 +1,30 @@
+# Copyright 2023-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Minimal test of PyMongo in an *Embedded mode* WSGI application.
+
+LoadModule wsgi_module ${MOD_WSGI_SO}
+
+# Avoid permissions issues
+WSGISocketPrefix /tmp/
+
+<VirtualHost *>
+    ServerName localhost
+    # Mount the script twice so that multiple interpreters are used.
+    # For the convenience of unittests, rather than hard-code the location of
+    # mod_wsgi_test.py, include it in the URL, so
+    # http://localhost/interpreter1/location-of-pymongo-checkout will work:
+    WSGIScriptAliasMatch ^/interpreter1/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py
+    WSGIScriptAliasMatch ^/interpreter2/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py
+</VirtualHost>

test/mod_wsgi_test/test_client.py

@@ -15,6 +15,7 @@
 """Test client for mod_wsgi application, see bug PYTHON-353."""
 
 import _thread as thread
+import random
 import sys
 import threading
 import time
@@ -24,7 +25,7 @@ from urllib.request import urlopen
 
 def parse_args():
     parser = OptionParser(
-        """usage: %prog [options] mode url
+        """usage: %prog [options] mode url [<url2>...]
 
   mode:\tparallel or serial"""
     )
@@ -37,7 +38,7 @@ def parse_args():
         type="int",
         dest="nrequests",
         default=50 * 1000,
-        help="Number of times to GET the URL, in total",
+        help="Number of times to GET the URLs, in total",
     )
 
     parser.add_option(
@@ -68,8 +69,9 @@ def parse_args():
     )
 
     try:
-        options, (mode, url) = parser.parse_args()
-    except ValueError:
+        options, args = parser.parse_args()
+        mode, urls = args[0], args[1:]
+    except (ValueError, IndexError):
         parser.print_usage()
         sys.exit(1)
 
@@ -77,10 +79,11 @@ def parse_args():
         parser.print_usage()
         sys.exit(1)
 
-    return options, mode, url
+    return options, mode, urls
 
 
-def get(url):
+def get(urls):
+    url = random.choice(urls)
     urlopen(url).read().strip()
 
 
@@ -89,17 +92,17 @@ class URLGetterThread(threading.Thread):
     counter_lock = threading.Lock()
     counter = 0
 
-    def __init__(self, options, url, nrequests_per_thread):
+    def __init__(self, options, urls, nrequests_per_thread):
         super().__init__()
         self.options = options
-        self.url = url
+        self.urls = urls
         self.nrequests_per_thread = nrequests_per_thread
         self.errors = 0
 
     def run(self):
         for _i in range(self.nrequests_per_thread):
             try:
-                get(url)
+                get(urls)
             except Exception as e:
                 print(e)
 
@@ -119,7 +122,7 @@ class URLGetterThread(threading.Thread):
                 print(counter)
 
 
-def main(options, mode, url):
+def main(options, mode, urls):
     start_time = time.time()
     errors = 0
     if mode == "parallel":
@@ -129,14 +132,14 @@ def main(options, mode, url):
             print(
                 "Getting {} {} times total in {} threads, "
                 "{} times per thread".format(
-                    url,
+                    urls,
                     nrequests_per_thread * options.nthreads,
                     options.nthreads,
                     nrequests_per_thread,
                 )
             )
         threads = [
-            URLGetterThread(options, url, nrequests_per_thread) for _ in range(options.nthreads)
+            URLGetterThread(options, urls, nrequests_per_thread) for _ in range(options.nthreads)
         ]
 
         for t in threads:
@@ -152,11 +155,11 @@ def main(options, mode, url):
     else:
         assert mode == "serial"
         if options.verbose:
-            print(f"Getting {url} {options.nrequests} times in one thread")
+            print(f"Getting {urls} {options.nrequests} times in one thread")
 
         for i in range(1, options.nrequests + 1):
             try:
-                get(url)
+                get(urls)
             except Exception as e:
                 print(e)
                 if not options.continue_:
@@ -179,5 +182,5 @@ def main(options, mode, url):
 
 
 if __name__ == "__main__":
-    options, mode, url = parse_args()
-    main(options, mode, url)
+    options, mode, urls = parse_args()
+    main(options, mode, urls)