From 9dda1346dd587500fcb83b5669ef16f31c1f7e54 Mon Sep 17 00:00:00 2001 From: "A. Jesse Jiryu Davis" Date: Tue, 23 Sep 2014 15:19:42 -0400 Subject: [PATCH] Test connecting to standalone or RS from the same mod_wsgi script. --- test/mod_wsgi_test/README.rst | 106 ++++++++++++++++++ test/mod_wsgi_test/mod_wsgi_test.conf | 9 +- ..._single_server.wsgi => mod_wsgi_test.wsgi} | 9 ++ .../mod_wsgi_test_replica_set.wsgi | 54 --------- 4 files changed, 118 insertions(+), 60 deletions(-) create mode 100644 test/mod_wsgi_test/README.rst rename test/mod_wsgi_test/{mod_wsgi_test_single_server.wsgi => mod_wsgi_test.wsgi} (83%) delete mode 100644 test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst new file mode 100644 index 000000000..128413a13 --- /dev/null +++ b/test/mod_wsgi_test/README.rst @@ -0,0 +1,106 @@ +Testing PyMongo with mod_wsgi +============================= + +These tests verify that PyMongo works with Apache and mod_wsgi. They are +primarily intended to prevent regression of +`PYTHON-353 `_, a connection leak +when PyMongo 2.2 was used with Python 2.6 and mod_wsgi 2.8. However, the test +may also catch concurrency bugs, or incompatibilities between PyMongo's C +extensions and the way mod_wsgi manages Python sub interpreters. It is +generally useful to test PyMongo in the unconventional environment that +mod_wsgi creates. + +Test Matrix +----------- + +PyMongo should be tested with several versions of mod_wsgi and a selection +of Python versions. Each combination of mod_wsgi and Python version should +be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi`` +detects if the deployment is a replica set and connects to the whole set. + +Setup +----- + +Compile Python +.............. + +We need a Python interpreter built as a shared library. Download the +source tarball for each Python version tested, untar it, and run:: + + ./configure --prefix=/some/directory --enable-shared + make + make install + +This results in an executable named "python" and a shared +library named something like "libpython2.7.so.1.0". +It may be necessary to add /some/directory/lib to your system's +LD_LIBRARY_PATH, or to make a symlink from your system's default library +directory to the shared library. For example, on Ubuntu:: + + ln -s /some/directory/lib/libpython2.7.so.1.0 /usr/lib64/ + +Compile mod_wsgi +................ + +Compile mod_wsgi for each combination for Python and mod_wsgi version in the +test matrix. For example, to compile mod_wsgi 3.4 for Python 2.6 on a +RedHat-like Linux:: + + sudo yum install -y httpd httpd-devel + wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz + tar xzf mod_wsgi-3.4.tar.gz + cd mod_wsgi-3.4 + ./configure --with-python=/some/directory/python + make + make install + +To ease testing of several matrix combinations, copy the resulting +``mod_wsgi.so`` to a safe place. + +Start mongod +............ + +Start a standalone listening on port 27017, or a replica set with a member +listening on port 27017. + +Configure Apache +................ + +Copy the appropriate version of ``mod_wsgi.so`` into Apache's modules +directory. Start Apache with the ``mod_wsgi_test.conf`` in this directory. + +Run the test +------------ + +Run the included ``test_client.py`` script:: + + python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \ + http://localhost/${WORKSPACE} + +...where the "n" argument is the total number of requests to make to Apache, +and "t" specifies the number of threads. ``WORKSPACE`` is the location of +the PyMongo checkout. + +Run this script again with different arguments to make serial requests:: + + python test/mod_wsgi_test/test_client.py -n 25000 serial \ + http://localhost/${WORKSPACE} + +The ``test_client.py`` script merely makes HTTP requests to Apache. Its +exit code is non-zero if any of its requests fails, for example with an +HTTP 500. + +The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`. +This script inserts some documents into MongoDB at startup, then queries +documents for each HTTP request. + +If PyMongo is leaking connections and "n" is much greater than the ulimit, +the test will fail when PyMongo exhausts its file descriptors. + +Automation +---------- + +At MongoDB, Inc. we use a Jenkins job that tests each combination in the +matrix. The job copies the appropriate version of ``mod_wsgi.so`` into +place, sets up Apache, starts a single server or replica set, +and runs ``test_client.py`` with the proper arguments. diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index 0b8c714af..8815f0b34 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -28,12 +28,9 @@ WSGISocketPrefix /tmp/ WSGIProcessGroup mod_wsgi_test # For the convienience of unittests, rather than hard-code the location of - # mod_wsgi_test_single_server.wsgi and mod_wsgi_test_replica_set.wsgi, - # include it in the URL, so - # http://localhost/single_server/location-of-pymongo-checkout will work: + # mod_wsgi_test.wsgi, include it in the URL, so + # http://localhost/location-of-pymongo-checkout will work: - WSGIScriptAliasMatch ^/single_server(.+) $1/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi - - WSGIScriptAliasMatch ^/replica_set(.+) $1/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi + WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi diff --git a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi similarity index 83% rename from test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi rename to test/mod_wsgi_test/mod_wsgi_test.wsgi index 995bc668a..6e4c1b16d 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_single_server.wsgi +++ b/test/mod_wsgi_test/mod_wsgi_test.wsgi @@ -26,9 +26,18 @@ sys.path.insert(0, repository_path) import pymongo from pymongo.mongo_client import MongoClient +from pymongo.mongo_replica_set_client import MongoReplicaSetClient # auto_start_request is part of the PYTHON-353 pathology client = MongoClient(auto_start_request=True) + +# If the deployment is a replica set, connect to the whole set. +replica_set_name = client.admin.command('ismaster').get('setName') +if replica_set_name: + client = MongoReplicaSetClient( + auto_start_request=True, + replicaSet=replica_set_name) + collection = client.test.test ndocs = 20 diff --git a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi b/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi deleted file mode 100644 index fef57611a..000000000 --- a/test/mod_wsgi_test/mod_wsgi_test_replica_set.wsgi +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012-2014 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Minimal test of PyMongo in a WSGI application with MongoReplicaSetClient, - see bug PYTHON-353. -""" - -import os -import sys - -this_path = os.path.dirname(os.path.join(os.getcwd(), __file__)) - -# Location of PyMongo checkout -repository_path = os.path.normpath(os.path.join(this_path, '..', '..')) -sys.path.insert(0, repository_path) - -import pymongo -from pymongo.mongo_replica_set_client import MongoReplicaSetClient - -# auto_start_request is part of the PYTHON-353 pathology -client = MongoReplicaSetClient(replicaSet='repl0', auto_start_request=True) -collection = client.test.test - -ndocs = 20 - -collection.drop() -collection.insert([{'i': i} for i in range(ndocs)]) -client.disconnect() # Discard main thread's request socket. - -try: - from mod_wsgi import version as mod_wsgi_version -except: - mod_wsgi_version = None - - -def application(environ, start_response): - results = list(collection.find().batch_size(10)) - assert len(results) == ndocs - output = 'python %s, mod_wsgi %s, pymongo %s' % ( - sys.version, mod_wsgi_version, pymongo.version) - response_headers = [('Content-Length', str(len(output)))] - start_response('200 OK', response_headers) - return [output]