PYTHON-1240 - Test mod_wsgi in Evergreen

.evergreen/config.yml

@@ -327,6 +327,15 @@ functions:
           ${PREPARE_SHELL}
           sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
 
+  "run mod_wsgi tests":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh
+
   "run tests":
     - command: shell.exec
       type: test
@@ -622,6 +631,24 @@ tasks:
             TOPOLOGY: "server"
         - func: "run enterprise auth tests"
 
+    - name: "mod-wsgi-standalone"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "server"
+        - func: "run mod_wsgi tests"
+
+    - name: "mod-wsgi-replica-set"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "replica_set"
+        - func: "run mod_wsgi tests"
+
 # }}}
 
 
@@ -1235,6 +1262,14 @@ buildvariants:
   tasks:
      - name: "test-enterprise-auth"
 
+- matrix_name: "tests-mod-wsgi"
+  matrix_spec: {"python-version": ["2.6", "3.6"]}
+  display_name: "mod_wsgi ${python-version}"
+  run_on: ubuntu1604-test
+  tasks:
+     - name: "mod-wsgi-standalone"
+     - name: "mod-wsgi-replica-set"
+
       # Platform notes
       # i386 builds of OpenSSL or Cyrus SASL are not available
       # Ubuntu14.04 only supports 2.6+ with SSL

.evergreen/run-mod-wsgi-tests.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -o xtrace
+set -o errexit
+
+# TODO: Rework mod_wsgi directory structure to avoid this mess.
+PYTHON_NAME=$(${PYTHON_BINARY} -c "import sys; print('python%s' % ('.'.join(str(val) for val in sys.version_info[:2])))")
+export MOD_WSGI_SO=/opt/python/mod_wsgi/4.5.13/${PYTHON_NAME}/mod_wsgi.so
+
+cd ..
+# TODO: Test with Apache 2.2 on Ubuntu 12.04 or Amazon Linux.
+apache2 -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache24ubuntu161404.conf
+trap "apache2 -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache24ubuntu161404.conf" EXIT HUP
+
+wget -O - "http://localhost:8080${PROJECT_DIRECTORY}"
+
+# Debug
+cat error_log
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY}
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY}
+

test/mod_wsgi_test/README.rst

@@ -27,17 +27,12 @@ Compile Python
 We need a Python interpreter built as a shared library. Download the
 source tarball for each Python version tested, untar it, and run::
 
-    ./configure --prefix=/some/directory --enable-shared
+    ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib"
     make
     make install
 
 This results in an executable named "python" or "python3" and a shared
 library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0".
-It may be necessary to add /some/directory/lib to your system's
-LD_LIBRARY_PATH, or to make a symlink from your system's default library
-directory to the shared library. For example, on Ubuntu::
-
-    ln -s /some/directory/lib/libpython3.3m.so.1.0 /usr/lib64/
 
 Compile mod_wsgi
 ................
@@ -50,7 +45,7 @@ RedHat-like Linux::
     wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz
     tar xzf mod_wsgi-3.4.tar.gz
     cd mod_wsgi-3.4
-    ./configure --with-python=/path/to/python-executable
+    ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib"
     make
     make install
 
@@ -66,8 +61,12 @@ listening on port 27017.
 Configure Apache
 ................
 
-Copy the appropriate version of ``mod_wsgi.so`` into Apache's modules
-directory. Start Apache with the ``mod_wsgi_test.conf`` in this directory.
+Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf``
+can find and load mod_wsgi.so::
+
+    export MOD_WSGI_SO=/path/to/mod_wsgi.so
+
+Start Apache with one of the config files in this directory.
 
 Run the test
 ------------
@@ -90,7 +89,7 @@ The ``test_client.py`` script merely makes HTTP requests to Apache. Its
 exit code is non-zero if any of its requests fails, for example with an
 HTTP 500.
 
-The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`.
+The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi``.
 This script inserts some documents into MongoDB at startup, then queries
 documents for each HTTP request.
 
@@ -100,7 +99,6 @@ the test will fail when PyMongo exhausts its file descriptors.
 Automation
 ----------
 
-At MongoDB, Inc. we use a Jenkins job that tests each combination in the
-matrix. The job copies the appropriate version of ``mod_wsgi.so`` into
-place, sets up Apache, starts a single server or replica set,
-and runs ``test_client.py`` with the proper arguments.
+At MongoDB, Inc. we use a continuous integration job that tests each
+combination in the matrix. The job starts up Apache, starts a single server
+or replica set, and runs ``test_client.py`` with the proper arguments.

test/mod_wsgi_test/apache22amazon.conf

@@ -0,0 +1,31 @@
+# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux
+ 
+# The modules directory is here by default.
+# ServerRoot "/etc/httpd"
+DocumentRoot ${PWD}
+PidFile ${PWD}/apache2.pid
+ 
+# Bind to localhost only, don't require sudo.
+Listen 127.0.0.1:8080
+ 
+# Required modules.
+LoadModule authz_host_module modules/mod_authz_host.so
+# Needed so we can set a custom log location.
+LoadModule log_config_module modules/mod_log_config.so
+ 
+ErrorLog ${PWD}/error_log
+CustomLog ${PWD}/access_log combined
+ 
+<Directory "/">
+    AllowOverride None
+    Order Deny,Allow
+    Deny from All
+</Directory>
+ 
+<Directory ${PWD}>
+    AllowOverride None
+    Order Allow,Deny
+    Allow from All
+</Directory>
+
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf

test/mod_wsgi_test/apache22ubuntu1204.conf

@@ -0,0 +1,29 @@
+# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04
+ 
+# The modules directory is here on Ubuntu.
+ServerRoot "/usr/lib/apache2"
+DocumentRoot ${PWD}
+PidFile ${PWD}/apache2.pid
+ 
+# Bind to localhost only, don't require sudo.
+Listen 127.0.0.1:8080
+ 
+# Required modules.
+LoadModule authz_host_module modules/mod_authz_host.so
+ 
+ErrorLog ${PWD}/error_log
+CustomLog ${PWD}/access_log combined
+ 
+<Directory "/">
+    AllowOverride None
+    Order Deny,Allow
+    Deny from All
+</Directory>
+
+<Directory ${PWD}>
+    AllowOverride None
+    Order Allow,Deny
+    Allow from All
+</Directory>
+ 
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf

test/mod_wsgi_test/apache24ubuntu161404.conf

@@ -0,0 +1,28 @@
+# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu 14.04/16.04
+ 
+# The modules directory is here on Ubuntu.
+ServerRoot "/usr/lib/apache2"
+DocumentRoot ${PWD}
+PidFile ${PWD}/apache2.pid
+ 
+# Bind to localhost only, don't require sudo.
+Listen 127.0.0.1:8080
+ 
+# Required modules.
+LoadModule mpm_prefork_module modules/mod_mpm_prefork.so
+LoadModule authz_core_module modules/mod_authz_core.so
+ 
+ErrorLog ${PWD}/error_log
+CustomLog ${PWD}/access_log combined
+ 
+<Directory "/">
+    AllowOverride None
+    Require all denied
+</Directory>
+
+<Directory ${PWD}>
+    AllowOverride None
+    Require all granted
+</Directory>
+ 
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf

test/mod_wsgi_test/mod_wsgi_test.conf

@@ -14,7 +14,7 @@
 
 # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353
 
-LoadModule wsgi_module modules/mod_wsgi.so
+LoadModule wsgi_module ${MOD_WSGI_SO}
 
 # Avoid permissions issues
 WSGISocketPrefix /tmp/

test/mod_wsgi_test/mod_wsgi_test.wsgi

@@ -51,7 +51,7 @@ except:
 def application(environ, start_response):
     results = list(collection.find().batch_size(10))
     assert len(results) == ndocs
-    output = 'python %s, mod_wsgi %s, pymongo %s' % (
+    output = ' python %s, mod_wsgi %s, pymongo %s ' % (
         sys.version, mod_wsgi_version, pymongo.version)
     response_headers = [('Content-Length', str(len(output)))]
     start_response('200 OK', response_headers)

test/mod_wsgi_test/test_client.py

@@ -118,7 +118,7 @@ def main(options, mode, url):
     start_time = time.time()
     errors = 0
     if mode == 'parallel':
-        nrequests_per_thread = options.nrequests / options.nthreads
+        nrequests_per_thread = options.nrequests // options.nthreads
 
         if options.verbose:
             print (