From f880111754f637c3448443a78c4b2125e9d4c709 Mon Sep 17 00:00:00 2001 From: Bernie Hackett Date: Mon, 13 Mar 2017 11:30:29 -0700 Subject: [PATCH] PYTHON-1240 - Test mod_wsgi in Evergreen --- .evergreen/config.yml | 35 ++++++++++++++++++++ .evergreen/run-mod-wsgi-tests.sh | 22 ++++++++++++ test/mod_wsgi_test/README.rst | 26 +++++++-------- test/mod_wsgi_test/apache22amazon.conf | 31 +++++++++++++++++ test/mod_wsgi_test/apache22ubuntu1204.conf | 29 ++++++++++++++++ test/mod_wsgi_test/apache24ubuntu161404.conf | 28 ++++++++++++++++ test/mod_wsgi_test/mod_wsgi_test.conf | 2 +- test/mod_wsgi_test/mod_wsgi_test.wsgi | 2 +- test/mod_wsgi_test/test_client.py | 2 +- 9 files changed, 160 insertions(+), 17 deletions(-) create mode 100644 .evergreen/run-mod-wsgi-tests.sh create mode 100644 test/mod_wsgi_test/apache22amazon.conf create mode 100644 test/mod_wsgi_test/apache22ubuntu1204.conf create mode 100644 test/mod_wsgi_test/apache24ubuntu161404.conf diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ee75b5b5c..58302d8b8 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -327,6 +327,15 @@ functions: ${PREPARE_SHELL} sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + "run mod_wsgi tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh + "run tests": - command: shell.exec type: test @@ -622,6 +631,24 @@ tasks: TOPOLOGY: "server" - func: "run enterprise auth tests" + - name: "mod-wsgi-standalone" + tags: ["mod_wsgi"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "server" + - func: "run mod_wsgi tests" + + - name: "mod-wsgi-replica-set" + tags: ["mod_wsgi"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "replica_set" + - func: "run mod_wsgi tests" + # }}} @@ -1235,6 +1262,14 @@ buildvariants: tasks: - name: "test-enterprise-auth" +- matrix_name: "tests-mod-wsgi" + matrix_spec: {"python-version": ["2.6", "3.6"]} + display_name: "mod_wsgi ${python-version}" + run_on: ubuntu1604-test + tasks: + - name: "mod-wsgi-standalone" + - name: "mod-wsgi-replica-set" + # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available # Ubuntu14.04 only supports 2.6+ with SSL diff --git a/.evergreen/run-mod-wsgi-tests.sh b/.evergreen/run-mod-wsgi-tests.sh new file mode 100644 index 000000000..a167ab2e1 --- /dev/null +++ b/.evergreen/run-mod-wsgi-tests.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -o xtrace +set -o errexit + +# TODO: Rework mod_wsgi directory structure to avoid this mess. +PYTHON_NAME=$(${PYTHON_BINARY} -c "import sys; print('python%s' % ('.'.join(str(val) for val in sys.version_info[:2])))") +export MOD_WSGI_SO=/opt/python/mod_wsgi/4.5.13/${PYTHON_NAME}/mod_wsgi.so + +cd .. +# TODO: Test with Apache 2.2 on Ubuntu 12.04 or Amazon Linux. +apache2 -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache24ubuntu161404.conf +trap "apache2 -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache24ubuntu161404.conf" EXIT HUP + +wget -O - "http://localhost:8080${PROJECT_DIRECTORY}" + +# Debug +cat error_log + +${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY} + +${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY} + diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index a77768828..5ca4e933e 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -27,17 +27,12 @@ Compile Python We need a Python interpreter built as a shared library. Download the source tarball for each Python version tested, untar it, and run:: - ./configure --prefix=/some/directory --enable-shared + ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib" make make install This results in an executable named "python" or "python3" and a shared library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0". -It may be necessary to add /some/directory/lib to your system's -LD_LIBRARY_PATH, or to make a symlink from your system's default library -directory to the shared library. For example, on Ubuntu:: - - ln -s /some/directory/lib/libpython3.3m.so.1.0 /usr/lib64/ Compile mod_wsgi ................ @@ -50,7 +45,7 @@ RedHat-like Linux:: wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz tar xzf mod_wsgi-3.4.tar.gz cd mod_wsgi-3.4 - ./configure --with-python=/path/to/python-executable + ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib" make make install @@ -66,8 +61,12 @@ listening on port 27017. Configure Apache ................ -Copy the appropriate version of ``mod_wsgi.so`` into Apache's modules -directory. Start Apache with the ``mod_wsgi_test.conf`` in this directory. +Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf`` +can find and load mod_wsgi.so:: + + export MOD_WSGI_SO=/path/to/mod_wsgi.so + +Start Apache with one of the config files in this directory. Run the test ------------ @@ -90,7 +89,7 @@ The ``test_client.py`` script merely makes HTTP requests to Apache. Its exit code is non-zero if any of its requests fails, for example with an HTTP 500. -The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`. +The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi``. This script inserts some documents into MongoDB at startup, then queries documents for each HTTP request. @@ -100,7 +99,6 @@ the test will fail when PyMongo exhausts its file descriptors. Automation ---------- -At MongoDB, Inc. we use a Jenkins job that tests each combination in the -matrix. The job copies the appropriate version of ``mod_wsgi.so`` into -place, sets up Apache, starts a single server or replica set, -and runs ``test_client.py`` with the proper arguments. +At MongoDB, Inc. we use a continuous integration job that tests each +combination in the matrix. The job starts up Apache, starts a single server +or replica set, and runs ``test_client.py`` with the proper arguments. diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf new file mode 100644 index 000000000..97ab0d997 --- /dev/null +++ b/test/mod_wsgi_test/apache22amazon.conf @@ -0,0 +1,31 @@ +# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux + +# The modules directory is here by default. +# ServerRoot "/etc/httpd" +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule authz_host_module modules/mod_authz_host.so +# Needed so we can set a custom log location. +LoadModule log_config_module modules/mod_log_config.so + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Order Deny,Allow + Deny from All + + + + AllowOverride None + Order Allow,Deny + Allow from All + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf new file mode 100644 index 000000000..9ff257bc3 --- /dev/null +++ b/test/mod_wsgi_test/apache22ubuntu1204.conf @@ -0,0 +1,29 @@ +# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04 + +# The modules directory is here on Ubuntu. +ServerRoot "/usr/lib/apache2" +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule authz_host_module modules/mod_authz_host.so + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Order Deny,Allow + Deny from All + + + + AllowOverride None + Order Allow,Deny + Allow from All + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf diff --git a/test/mod_wsgi_test/apache24ubuntu161404.conf b/test/mod_wsgi_test/apache24ubuntu161404.conf new file mode 100644 index 000000000..24ad25f8a --- /dev/null +++ b/test/mod_wsgi_test/apache24ubuntu161404.conf @@ -0,0 +1,28 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu 14.04/16.04 + +# The modules directory is here on Ubuntu. +ServerRoot "/usr/lib/apache2" +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index 987d93e8e..9505933e9 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -14,7 +14,7 @@ # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 -LoadModule wsgi_module modules/mod_wsgi.so +LoadModule wsgi_module ${MOD_WSGI_SO} # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/mod_wsgi_test/mod_wsgi_test.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi index fb97d32c2..9b435b1ed 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.wsgi +++ b/test/mod_wsgi_test/mod_wsgi_test.wsgi @@ -51,7 +51,7 @@ except: def application(environ, start_response): results = list(collection.find().batch_size(10)) assert len(results) == ndocs - output = 'python %s, mod_wsgi %s, pymongo %s' % ( + output = ' python %s, mod_wsgi %s, pymongo %s ' % ( sys.version, mod_wsgi_version, pymongo.version) response_headers = [('Content-Length', str(len(output)))] start_response('200 OK', response_headers) diff --git a/test/mod_wsgi_test/test_client.py b/test/mod_wsgi_test/test_client.py index 596857e46..6ef807287 100644 --- a/test/mod_wsgi_test/test_client.py +++ b/test/mod_wsgi_test/test_client.py @@ -118,7 +118,7 @@ def main(options, mode, url): start_time = time.time() errors = 0 if mode == 'parallel': - nrequests_per_thread = options.nrequests / options.nthreads + nrequests_per_thread = options.nrequests // options.nthreads if options.verbose: print (