PYTHON-4441 Use deferred imports instead of lazy module loading (#1648)

doc/changelog.rst

@@ -8,6 +8,22 @@ The handshake metadata for "os.name" on Windows has been simplified to "Windows"
 
 .. warning:: PyMongo 4.8 drops support for Python 3.7 and PyPy 3.8: Python 3.8+ or PyPy 3.9+ is now required.
 
+Changes in Version 4.7.3
+-------------------------
+
+Version 4.7.3 has further fixes for lazily loading modules.
+
+- Use deferred imports instead of importlib lazy module loading.
+- Improve import time on Windows.
+
+Issues Resolved
+...............
+
+See the `PyMongo 4.7.3 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 4.7.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=39865
+
 Changes in Version 4.7.2
 -------------------------
 

pymongo/_gcp_helpers.py

@@ -16,10 +16,11 @@
 from __future__ import annotations
 
 from typing import Any
-from urllib.request import Request, urlopen
 
 
 def _get_gcp_response(resource: str, timeout: float = 5) -> dict[str, Any]:
+    from urllib.request import Request, urlopen
+
     url = "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity"
     url += f"?audience={resource}"
     headers = {"Metadata-Flavor": "Google"}

pymongo/_lazy_import.py

@@ -1,43 +0,0 @@
-# Copyright 2024-present MongoDB, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-from __future__ import annotations
-
-import importlib.util
-import sys
-from types import ModuleType
-
-
-def lazy_import(name: str) -> ModuleType:
-    """Lazily import a module by name
-
-    From https://docs.python.org/3/library/importlib.html#implementing-lazy-imports
-    """
-    # Workaround for PYTHON-4424.
-    if "__compiled__" in globals():
-        return importlib.import_module(name)
-    try:
-        spec = importlib.util.find_spec(name)
-    except ValueError:
-        # Note: this cannot be ModuleNotFoundError, see PYTHON-4424.
-        raise ImportError(name=name) from None
-    if spec is None:
-        # Note: this cannot be ModuleNotFoundError, see PYTHON-4424.
-        raise ImportError(name=name)
-    assert spec is not None
-    loader = importlib.util.LazyLoader(spec.loader)  # type:ignore[arg-type]
-    spec.loader = loader
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[name] = module
-    loader.exec_module(module)
-    return module

pymongo/auth_aws.py

@@ -15,15 +15,6 @@
 """MONGODB-AWS Authentication helpers."""
 from __future__ import annotations
 
-from pymongo._lazy_import import lazy_import
-
-try:
-    pymongo_auth_aws = lazy_import("pymongo_auth_aws")
-    _HAVE_MONGODB_AWS = True
-except ImportError:
-    _HAVE_MONGODB_AWS = False
-
-
 from typing import TYPE_CHECKING, Any, Mapping, Type
 
 import bson
@@ -38,11 +29,13 @@ if TYPE_CHECKING:
 
 def _authenticate_aws(credentials: MongoCredential, conn: Connection) -> None:
     """Authenticate using MONGODB-AWS."""
-    if not _HAVE_MONGODB_AWS:
+    try:
+        import pymongo_auth_aws  # type:ignore[import]
+    except ImportError as e:
         raise ConfigurationError(
             "MONGODB-AWS authentication requires pymongo-auth-aws: "
             "install with: python -m pip install 'pymongo[aws]'"
-        )
+        ) from e
 
     # Delayed import.
     from pymongo_auth_aws.auth import (  # type:ignore[import]

pymongo/client_options.py

@@ -19,7 +19,6 @@ from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence, cast
 
 from bson.codec_options import _parse_codec_options
 from pymongo import common
-from pymongo.auth import MongoCredential, _build_credentials_tuple
 from pymongo.compression_support import CompressionSettings
 from pymongo.errors import ConfigurationError
 from pymongo.monitoring import _EventListener, _EventListeners
@@ -36,6 +35,7 @@ from pymongo.write_concern import WriteConcern, validate_boolean
 
 if TYPE_CHECKING:
     from bson.codec_options import CodecOptions
+    from pymongo.auth import MongoCredential
     from pymongo.encryption_options import AutoEncryptionOpts
     from pymongo.pyopenssl_context import SSLContext
     from pymongo.topology_description import _ServerSelector
@@ -48,6 +48,8 @@ def _parse_credentials(
     mechanism = options.get("authmechanism", "DEFAULT" if username else None)
     source = options.get("authsource")
     if username or mechanism:
+        from pymongo.auth import _build_credentials_tuple
+
         return _build_credentials_tuple(mechanism, source, username, password, options, database)
     return None
 

pymongo/common.py

@@ -40,8 +40,6 @@ from bson import SON
 from bson.binary import UuidRepresentation
 from bson.codec_options import CodecOptions, DatetimeConversion, TypeRegistry
 from bson.raw_bson import RawBSONDocument
-from pymongo.auth import MECHANISMS
-from pymongo.auth_oidc import OIDCCallback
 from pymongo.compression_support import (
     validate_compressors,
     validate_zlib_compression_level,
@@ -380,6 +378,8 @@ def validate_read_preference_mode(dummy: Any, value: Any) -> _ServerMode:
 
 def validate_auth_mechanism(option: str, value: Any) -> str:
     """Validate the authMechanism URI option."""
+    from pymongo.auth import MECHANISMS
+
     if value not in MECHANISMS:
         raise ValueError(f"{option} must be in {tuple(MECHANISMS)}")
     return value
@@ -444,6 +444,8 @@ def validate_auth_mechanism_properties(option: str, value: Any) -> dict[str, Uni
             elif key in ["ALLOWED_HOSTS"] and isinstance(value, list):
                 props[key] = value
             elif key in ["OIDC_CALLBACK", "OIDC_HUMAN_CALLBACK"]:
+                from pymongo.auth_oidc import OIDCCallback
+
                 if not isinstance(value, OIDCCallback):
                     raise ValueError("callback must be an OIDCCallback object")
                 props[key] = value

pymongo/compression_support.py

@@ -16,36 +16,41 @@ from __future__ import annotations
 import warnings
 from typing import Any, Iterable, Optional, Union
 
-from pymongo._lazy_import import lazy_import
 from pymongo.hello import HelloCompat
-from pymongo.monitoring import _SENSITIVE_COMMANDS
-
-try:
-    snappy = lazy_import("snappy")
-    _HAVE_SNAPPY = True
-except ImportError:
-    # python-snappy isn't available.
-    _HAVE_SNAPPY = False
-
-try:
-    zlib = lazy_import("zlib")
-
-    _HAVE_ZLIB = True
-except ImportError:
-    # Python built without zlib support.
-    _HAVE_ZLIB = False
-
-try:
-    zstandard = lazy_import("zstandard")
-    _HAVE_ZSTD = True
-except ImportError:
-    _HAVE_ZSTD = False
+from pymongo.helpers import _SENSITIVE_COMMANDS
 
 _SUPPORTED_COMPRESSORS = {"snappy", "zlib", "zstd"}
 _NO_COMPRESSION = {HelloCompat.CMD, HelloCompat.LEGACY_CMD}
 _NO_COMPRESSION.update(_SENSITIVE_COMMANDS)
 
 
+def _have_snappy() -> bool:
+    try:
+        import snappy  # type:ignore[import]  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _have_zlib() -> bool:
+    try:
+        import zlib  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _have_zstd() -> bool:
+    try:
+        import zstandard  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
 def validate_compressors(dummy: Any, value: Union[str, Iterable[str]]) -> list[str]:
     try:
         # `value` is string.
@@ -58,21 +63,21 @@ def validate_compressors(dummy: Any, value: Union[str, Iterable[str]]) -> list[s
         if compressor not in _SUPPORTED_COMPRESSORS:
             compressors.remove(compressor)
             warnings.warn(f"Unsupported compressor: {compressor}", stacklevel=2)
-        elif compressor == "snappy" and not _HAVE_SNAPPY:
+        elif compressor == "snappy" and not _have_snappy():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with snappy is not available. "
                 "You must install the python-snappy module for snappy support.",
                 stacklevel=2,
             )
-        elif compressor == "zlib" and not _HAVE_ZLIB:
+        elif compressor == "zlib" and not _have_zlib():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with zlib is not available. "
                 "The zlib module is not available.",
                 stacklevel=2,
             )
-        elif compressor == "zstd" and not _HAVE_ZSTD:
+        elif compressor == "zstd" and not _have_zstd():
             compressors.remove(compressor)
             warnings.warn(
                 "Wire protocol compression with zstandard is not available. "
@@ -117,6 +122,8 @@ class SnappyContext:
 
     @staticmethod
     def compress(data: bytes) -> bytes:
+        import snappy
+
         return snappy.compress(data)
 
 
@@ -127,6 +134,8 @@ class ZlibContext:
         self.level = level
 
     def compress(self, data: bytes) -> bytes:
+        import zlib
+
         return zlib.compress(data, self.level)
 
 
@@ -137,6 +146,8 @@ class ZstdContext:
     def compress(data: bytes) -> bytes:
         # ZstdCompressor is not thread safe.
         # TODO: Use a pool?
+        import zstandard
+
         return zstandard.ZstdCompressor().compress(data)
 
 
@@ -146,12 +157,18 @@ def decompress(data: bytes, compressor_id: int) -> bytes:
         # https://github.com/andrix/python-snappy/issues/65
         # This only matters when data is a memoryview since
         # id(bytes(data)) == id(data) when data is a bytes.
+        import snappy
+
         return snappy.uncompress(bytes(data))
     elif compressor_id == ZlibContext.compressor_id:
+        import zlib
+
         return zlib.decompress(data)
     elif compressor_id == ZstdContext.compressor_id:
         # ZstdDecompressor is not thread safe.
         # TODO: Use a pool?
+        import zstandard
+
         return zstandard.ZstdDecompressor().decompress(data)
     else:
         raise ValueError("Unknown compressorId %d" % (compressor_id,))

pymongo/helpers.py

@@ -93,6 +93,21 @@ _REAUTHENTICATION_REQUIRED_CODE: int = 391
 # Server code raised when authentication fails.
 _AUTHENTICATION_FAILURE_CODE: int = 18
 
+# Note - to avoid bugs from forgetting which if these is all lowercase and
+# which are camelCase, and at the same time avoid having to add a test for
+# every command, use all lowercase here and test against command_name.lower().
+_SENSITIVE_COMMANDS: set = {
+    "authenticate",
+    "saslstart",
+    "saslcontinue",
+    "getnonce",
+    "createuser",
+    "updateuser",
+    "copydbgetnonce",
+    "copydbsaslstart",
+    "copydb",
+}
+
 
 def _gen_index_name(keys: _IndexList) -> str:
     """Generate an index name from the set of fields it is over."""

pymongo/monitoring.py

@@ -191,7 +191,7 @@ from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence
 
 from bson.objectid import ObjectId
 from pymongo.hello import Hello, HelloCompat
-from pymongo.helpers import _handle_exception
+from pymongo.helpers import _SENSITIVE_COMMANDS, _handle_exception
 from pymongo.typings import _Address, _DocumentOut
 
 if TYPE_CHECKING:
@@ -507,22 +507,6 @@ def register(listener: _EventListener) -> None:
         _LISTENERS.cmap_listeners.append(listener)
 
 
-# Note - to avoid bugs from forgetting which if these is all lowercase and
-# which are camelCase, and at the same time avoid having to add a test for
-# every command, use all lowercase here and test against command_name.lower().
-_SENSITIVE_COMMANDS: set = {
-    "authenticate",
-    "saslstart",
-    "saslcontinue",
-    "getnonce",
-    "createuser",
-    "updateuser",
-    "copydbgetnonce",
-    "copydbsaslstart",
-    "copydb",
-}
-
-
 # The "hello" command is also deemed sensitive when attempting speculative
 # authentication.
 def _is_speculative_authenticate(command_name: str, doc: Mapping[str, Any]) -> bool:

pymongo/pool.py

@@ -41,7 +41,7 @@ from typing import (
 
 import bson
 from bson import DEFAULT_CODEC_OPTIONS
-from pymongo import __version__, _csot, auth, helpers
+from pymongo import __version__, _csot, helpers
 from pymongo.client_session import _validate_session_write_concern
 from pymongo.common import (
     MAX_BSON_SIZE,
@@ -860,6 +860,8 @@ class Connection:
         if creds:
             if creds.mechanism == "DEFAULT" and creds.username:
                 cmd["saslSupportedMechs"] = creds.source + "." + creds.username
+            from pymongo import auth
+
             auth_ctx = auth._AuthContext.from_credentials(creds, self.address)
             if auth_ctx:
                 speculative_authenticate = auth_ctx.speculate_command()
@@ -1091,6 +1093,8 @@ class Connection:
         if not self.ready:
             creds = self.opts._credentials
             if creds:
+                from pymongo import auth
+
                 auth.authenticate(creds, self, reauthenticate=reauthenticate)
             self.ready = True
             if self.enabled_for_cmap:

pymongo/pyopenssl_context.py

@@ -25,10 +25,11 @@ from errno import EINTR as _EINTR
 from ipaddress import ip_address as _ip_address
 from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
 
+import cryptography.x509 as x509
+import service_identity
 from OpenSSL import SSL as _SSL
 from OpenSSL import crypto as _crypto
 
-from pymongo._lazy_import import lazy_import
 from pymongo.errors import ConfigurationError as _ConfigurationError
 from pymongo.errors import _CertificateError  # type:ignore[attr-defined]
 from pymongo.ocsp_cache import _OCSPCache
@@ -37,14 +38,9 @@ from pymongo.socket_checker import SocketChecker as _SocketChecker
 from pymongo.socket_checker import _errno_from_exception
 from pymongo.write_concern import validate_boolean
 
-_x509 = lazy_import("cryptography.x509")
-_service_identity = lazy_import("service_identity")
-_service_identity_pyopenssl = lazy_import("service_identity.pyopenssl")
-
 if TYPE_CHECKING:
     from ssl import VerifyMode
 
-    from cryptography.x509 import Certificate
 
 _T = TypeVar("_T")
 
@@ -184,7 +180,7 @@ class _CallbackData:
     """Data class which is passed to the OCSP callback."""
 
     def __init__(self) -> None:
-        self.trusted_ca_certs: Optional[list[Certificate]] = None
+        self.trusted_ca_certs: Optional[list[x509.Certificate]] = None
         self.check_ocsp_endpoint: Optional[bool] = None
         self.ocsp_response_cache = _OCSPCache()
 
@@ -336,11 +332,12 @@ class SSLContext:
         """Attempt to load CA certs from Windows trust store."""
         cert_store = self._ctx.get_cert_store()
         oid = _stdlibssl.Purpose.SERVER_AUTH.oid
+
         for cert, encoding, trust in _stdlibssl.enum_certificates(store):  # type: ignore
             if encoding == "x509_asn":
                 if trust is True or oid in trust:
                     cert_store.add_cert(
-                        _crypto.X509.from_cryptography(_x509.load_der_x509_certificate(cert))
+                        _crypto.X509.from_cryptography(x509.load_der_x509_certificate(cert))
                     )
 
     def load_default_certs(self) -> None:
@@ -404,14 +401,16 @@ class SSLContext:
             # XXX: Do this in a callback registered with
             # SSLContext.set_info_callback? See Twisted for an example.
             if self.check_hostname and server_hostname is not None:
+                from service_identity import pyopenssl
+
                 try:
                     if _is_ip_address(server_hostname):
-                        _service_identity_pyopenssl.verify_ip_address(ssl_conn, server_hostname)
+                        pyopenssl.verify_ip_address(ssl_conn, server_hostname)
                     else:
-                        _service_identity_pyopenssl.verify_hostname(ssl_conn, server_hostname)
-                except (
-                    _service_identity.SICertificateError,
-                    _service_identity.SIVerificationError,
+                        pyopenssl.verify_hostname(ssl_conn, server_hostname)
+                except (  # type:ignore[misc]
+                    service_identity.SICertificateError,
+                    service_identity.SIVerificationError,
                 ) as exc:
                     raise _CertificateError(str(exc)) from None
         return ssl_conn

pymongo/srv_resolver.py

@@ -17,17 +17,22 @@ from __future__ import annotations
 
 import ipaddress
 import random
-from typing import Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 from pymongo.common import CONNECT_TIMEOUT
 from pymongo.errors import ConfigurationError
 
-try:
+if TYPE_CHECKING:
     from dns import resolver
 
-    _HAVE_DNSPYTHON = True
-except ImportError:
-    _HAVE_DNSPYTHON = False
+
+def _have_dnspython() -> bool:
+    try:
+        import dns  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
 
 
 # dnspython can return bytes or str from various parts
@@ -40,6 +45,8 @@ def maybe_decode(text: Union[str, bytes]) -> str:
 
 # PYTHON-2667 Lazily call dns.resolver methods for compatibility with eventlet.
 def _resolve(*args: Any, **kwargs: Any) -> resolver.Answer:
+    from dns import resolver
+
     if hasattr(resolver, "resolve"):
         # dnspython >= 2
         return resolver.resolve(*args, **kwargs)
@@ -81,6 +88,8 @@ class _SrvResolver:
             raise ConfigurationError(_INVALID_HOST_MSG % (fqdn,))
 
     def get_options(self) -> Optional[str]:
+        from dns import resolver
+
         try:
             results = _resolve(self.__fqdn, "TXT", lifetime=self.__connect_timeout)
         except (resolver.NoAnswer, resolver.NXDOMAIN):

pymongo/uri_parser.py

@@ -40,7 +40,7 @@ from pymongo.common import (
     get_validated_options,
 )
 from pymongo.errors import ConfigurationError, InvalidURI
-from pymongo.srv_resolver import _HAVE_DNSPYTHON, _SrvResolver
+from pymongo.srv_resolver import _have_dnspython, _SrvResolver
 from pymongo.typings import _Address
 
 if TYPE_CHECKING:
@@ -472,7 +472,7 @@ def parse_uri(
         is_srv = False
         scheme_free = uri[SCHEME_LEN:]
     elif uri.startswith(SRV_SCHEME):
-        if not _HAVE_DNSPYTHON:
+        if not _have_dnspython():
             python_path = sys.executable or "python"
             raise ConfigurationError(
                 'The "dnspython" module must be '

test/test_client.py

@@ -86,7 +86,7 @@ from pymongo import event_loggers, message, monitoring
 from pymongo.client_options import ClientOptions
 from pymongo.command_cursor import CommandCursor
 from pymongo.common import _UUID_REPRESENTATIONS, CONNECT_TIMEOUT
-from pymongo.compression_support import _HAVE_SNAPPY, _HAVE_ZSTD
+from pymongo.compression_support import _have_snappy, _have_zstd
 from pymongo.cursor import Cursor, CursorType
 from pymongo.database import Database
 from pymongo.driver_info import DriverInfo
@@ -1558,7 +1558,7 @@ class TestClient(IntegrationTest):
         self.assertEqual(opts.compressors, ["zlib"])
         self.assertEqual(opts.zlib_compression_level, -1)
 
-        if not _HAVE_SNAPPY:
+        if not _have_snappy():
             uri = "mongodb://localhost:27017/?compressors=snappy"
             client = MongoClient(uri, connect=False)
             opts = compression_settings(client)
@@ -1573,7 +1573,7 @@ class TestClient(IntegrationTest):
             opts = compression_settings(client)
             self.assertEqual(opts.compressors, ["snappy", "zlib"])
 
-        if not _HAVE_ZSTD:
+        if not _have_zstd():
             uri = "mongodb://localhost:27017/?compressors=zstd"
             client = MongoClient(uri, connect=False)
             opts = compression_settings(client)

test/test_srv_polling.py

@@ -28,7 +28,7 @@ import pymongo
 from pymongo import common
 from pymongo.errors import ConfigurationError
 from pymongo.mongo_client import MongoClient
-from pymongo.srv_resolver import _HAVE_DNSPYTHON
+from pymongo.srv_resolver import _have_dnspython
 
 WAIT_TIME = 0.1
 
@@ -148,7 +148,7 @@ class TestSrvPolling(unittest.TestCase):
         return True
 
     def run_scenario(self, dns_response, expect_change):
-        self.assertEqual(_HAVE_DNSPYTHON, True)
+        self.assertEqual(_have_dnspython(), True)
         if callable(dns_response):
             dns_resolver_response = dns_response
         else:

test/test_uri_spec.py

@@ -27,7 +27,7 @@ sys.path[0:0] = [""]
 from test import clear_warning_registry, unittest
 
 from pymongo.common import INTERNAL_URI_OPTION_NAME_MAP, validate
-from pymongo.compression_support import _HAVE_SNAPPY
+from pymongo.compression_support import _have_snappy
 from pymongo.uri_parser import SRV_SCHEME, parse_uri
 
 CONN_STRING_TEST_PATH = os.path.join(
@@ -95,7 +95,7 @@ def run_scenario_in_dir(target_workdir):
 def create_test(test, test_workdir):
     def run_scenario(self):
         compressors = (test.get("options") or {}).get("compressors", [])
-        if "snappy" in compressors and not _HAVE_SNAPPY:
+        if "snappy" in compressors and not _have_snappy():
             self.skipTest("This test needs the snappy module.")
         valid = True
         warning = False

test/utils.py

@@ -39,9 +39,9 @@ from pymongo.collection import ReturnDocument
 from pymongo.cursor import CursorType
 from pymongo.errors import ConfigurationError, OperationFailure
 from pymongo.hello import HelloCompat
+from pymongo.helpers import _SENSITIVE_COMMANDS
 from pymongo.lock import _create_lock
 from pymongo.monitoring import (
-    _SENSITIVE_COMMANDS,
     ConnectionCheckedInEvent,
     ConnectionCheckedOutEvent,
     ConnectionCheckOutFailedEvent,