From eda4fbb1591bd88d58d5bd3452f82ed656e95b1c Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 18 Nov 2019 18:27:01 -0800 Subject: [PATCH] PYTHON-2043 Spawn mongocryptd as a daemon process and silence resource warnings --- pymongo/daemon.py | 144 ++++++++++++++++++++++++++++++++++++++++++ pymongo/encryption.py | 5 +- 2 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 pymongo/daemon.py diff --git a/pymongo/daemon.py b/pymongo/daemon.py new file mode 100644 index 000000000..f066a02c2 --- /dev/null +++ b/pymongo/daemon.py @@ -0,0 +1,144 @@ +# Copyright 2019-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Support for spawning a daemon process. + +PyMongo only attempts to spawn the mongocryptd daemon process when automatic +client-side field level encryption is enabled. See +:ref:`automatic-client-side-encryption` for more info. +""" + +import os +import subprocess +import sys +import time + +# The maximum amount of time to wait for the intermediate subprocess. +_WAIT_TIMEOUT = 10 +_THIS_FILE = os.path.realpath(__file__) + +if sys.version_info[0] < 3: + def _popen_wait(popen, timeout): + """Implement wait timeout support for Python 2.""" + from pymongo.monotonic import time as _time + deadline = _time() + timeout + # Initial delay of 1ms + delay = .0005 + while True: + returncode = popen.poll() + if returncode is not None: + return returncode + + remaining = deadline - _time() + if remaining <= 0: + # Just return None instead of raising an error. + return None + delay = min(delay * 2, remaining, .5) + time.sleep(delay) + +else: + def _popen_wait(popen, timeout): + """Implement wait timeout support for Python 3.""" + try: + return popen.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Silence TimeoutExpired errors. + return None + + +def _silence_resource_warning(popen): + """Silence Popen's ResourceWarning. + + Note this should only be used if the process was created as a daemon. + """ + # Set the returncode to avoid this warning when popen is garbage collected: + # "ResourceWarning: subprocess XXX is still running". + # See https://bugs.python.org/issue38890 and + # https://bugs.python.org/issue26741. + popen.returncode = 0 + + +if sys.platform == 'win32': + # On Windows we spawn the daemon process simply by using DETACHED_PROCESS. + _DETACHED_PROCESS = getattr(subprocess, 'DETACHED_PROCESS', 0x00000008) + + def _spawn_daemon(args): + """Spawn a daemon process (Windows).""" + with open(os.devnull, 'r+b') as devnull: + popen = subprocess.Popen( + args, + creationflags=_DETACHED_PROCESS, + stdin=devnull, stderr=devnull, stdout=devnull) + _silence_resource_warning(popen) +else: + # On Unix we spawn the daemon process with a double Popen. + # 1) The first Popen runs this file as a Python script using the current + # interpreter. + # 2) The script then decouples itself and performs the second Popen to + # spawn the daemon process. + # 3) The original process waits up to 10 seconds for the script to exit. + # + # Note that we do not call fork() directly because we want this procedure + # to be safe to call from any thread. Using Popen instead of fork also + # avoids triggering the application's os.register_at_fork() callbacks when + # we spawn the mongocryptd daemon process. + def _spawn(args): + """Spawn the process and silence stdout/stderr.""" + with open(os.devnull, 'r+b') as devnull: + return subprocess.Popen( + args, + close_fds=True, + stdin=devnull, stderr=devnull, stdout=devnull) + + + def _spawn_daemon_double_popen(args): + """Spawn a daemon process using a double subprocess.Popen.""" + spawner_args = [sys.executable, _THIS_FILE] + spawner_args.extend(args) + temp_proc = subprocess.Popen(spawner_args, close_fds=True) + # Reap the intermediate child process to avoid creating zombie + # processes. + _popen_wait(temp_proc, _WAIT_TIMEOUT) + + + def _spawn_daemon(args): + """Spawn a daemon process (Unix).""" + # "If Python is unable to retrieve the real path to its executable, + # sys.executable will be an empty string or None". + if sys.executable: + _spawn_daemon_double_popen(args) + else: + # Fallback to spawn a non-daemon process without silencing the + # resource warning. We do not use fork here because it is not + # safe to call from a thread on all systems. + # Unfortunately, this means that: + # 1) If the parent application is killed via Ctrl-C, the + # non-daemon process will also be killed. + # 2) Each non-daemon process will hang around as a zombie process + # until the main application exits. + _spawn(args) + + + if __name__ == '__main__': + # Attempt to start a new session to decouple from the parent. + if hasattr(os, 'setsid'): + try: + os.setsid() + except OSError: + pass + + # We are performing a double fork (Popen) to spawn the process as a + # daemon so it is safe to ignore the resource warning. + _silence_resource_warning(_spawn(sys.argv[1:])) + os._exit(0) diff --git a/pymongo/encryption.py b/pymongo/encryption.py index e782a9f13..f71cd48b6 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -52,6 +52,7 @@ from pymongo.read_concern import ReadConcern from pymongo.ssl_support import get_ssl_context from pymongo.uri_parser import parse_host from pymongo.write_concern import WriteConcern +from pymongo.daemon import _spawn_daemon _HTTPS_PORT = 443 @@ -146,9 +147,7 @@ class _EncryptionIO(MongoCryptCallback): self._spawned = True args = [self.opts._mongocryptd_spawn_path or 'mongocryptd'] args.extend(self.opts._mongocryptd_spawn_args) - # Silence mongocryptd output, users should pass --logpath. - with open(os.devnull, 'wb') as devnull: - subprocess.Popen(args, stdout=devnull, stderr=devnull) + _spawn_daemon(args) def mark_command(self, database, cmd): """Mark a command for encryption.