From 5dda2aa30692e8529555f57efa7bcd203a7d5ac1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 17:38:16 +0000 Subject: [PATCH 1/4] Just use default `safe=...` characters for `urlescape` (#3376) --- httpx/_urls.py | 16 +--------------- tests/models/test_url.py | 4 ++-- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/httpx/_urls.py b/httpx/_urls.py index bfc0e9e6..7976cb18 100644 --- a/httpx/_urls.py +++ b/httpx/_urls.py @@ -12,20 +12,6 @@ from ._utils import primitive_value_to_str __all__ = ["URL", "QueryParams"] -# To urlencode query parameters, we use the whatwg query percent-encode set -# and additionally escape U+0025 (%), U+0026 (&), U+002B (+) and U+003D (=). - -# https://url.spec.whatwg.org/#percent-encoded-bytes - -URLENCODE_SAFE = "".join( - [ - chr(i) - for i in range(0x20, 0x7F) - if i not in (0x20, 0x22, 0x23, 0x25, 0x26, 0x2B, 0x3C, 0x3D, 0x3E) - ] -) - - class URL: """ url = httpx.URL("HTTPS://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink") @@ -619,7 +605,7 @@ class QueryParams(typing.Mapping[str, str]): return sorted(self.multi_items()) == sorted(other.multi_items()) def __str__(self) -> str: - return urlencode(self.multi_items(), safe=URLENCODE_SAFE) + return urlencode(self.multi_items()) def __repr__(self) -> str: class_name = self.__class__.__name__ diff --git a/tests/models/test_url.py b/tests/models/test_url.py index d32ed521..03072e8f 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -148,7 +148,7 @@ def test_url_query_encoding(): assert url.raw_path == b"/?a=b+c&d=e/f" url = httpx.URL("https://www.example.com/", params={"a": "b c", "d": "e/f"}) - assert url.raw_path == b"/?a=b+c&d=e/f" + assert url.raw_path == b"/?a=b+c&d=e%2Ff" def test_url_params(): @@ -309,7 +309,7 @@ def test_param_with_existing_escape_requires_encoding(): # even if they include a valid escape sequence. # We want to match browser form behaviour here. url = httpx.URL("http://webservice", params={"u": "http://example.com?q=foo%2Fa"}) - assert str(url) == "http://webservice?u=http://example.com?q%3Dfoo%252Fa" + assert str(url) == "http://webservice?u=http%3A%2F%2Fexample.com%3Fq%3Dfoo%252Fa" # Tests for query parameter percent encoding. From eeb5e3c2a3ff2403ec47b5926715ecd61143d92d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 17:38:33 +0000 Subject: [PATCH 2/4] Cleanup unneccessary test case (#3375) --- tests/client/test_auth.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/client/test_auth.py b/tests/client/test_auth.py index b3aeaf4e..7638b8bd 100644 --- a/tests/client/test_auth.py +++ b/tests/client/test_auth.py @@ -270,29 +270,6 @@ def test_netrc_auth_credentials_do_not_exist() -> None: assert response.json() == {"auth": None} -@pytest.mark.skipif( - sys.version_info < (3, 11), - reason="netrc files without a password are invalid with Python < 3.11", -) -def test_netrc_auth_nopassword() -> None: # pragma: no cover - """ - Python has different netrc parsing behaviours with different versions. - For Python 3.11+ a netrc file with no password is valid. In this case - we want to check that we allow the netrc auth, and simply don't provide - any credentials in the request. - """ - netrc_file = str(FIXTURES_DIR / ".netrc-nopassword") - url = "http://example.org" - app = App() - auth = httpx.NetRCAuth(netrc_file) - - with httpx.Client(transport=httpx.MockTransport(app), auth=auth) as client: - response = client.get(url) - - assert response.status_code == 200 - assert response.json() == {"auth": None} - - @pytest.mark.skipif( sys.version_info >= (3, 11), reason="netrc files without a password are valid from Python >= 3.11", From e9cabc8e1deb6548c5c5e2f453e343d39a2a5321 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Tue, 29 Oct 2024 13:18:39 +0000 Subject: [PATCH 3/4] made dependencies on certifi and httpcore only load when required (#3377) Co-authored-by: Tom Christie --- CHANGELOG.md | 5 ++-- httpx/__version__.py | 2 +- httpx/_config.py | 4 +-- httpx/_main.py | 4 ++- httpx/_transports/default.py | 56 +++++++++++++++++++++++------------- tests/test_api.py | 15 ++++++++++ tests/test_config.py | 15 ++++++++++ 7 files changed, 75 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53aaa6e0..1d32e53d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## Version 0.28.0 +## [Unreleased] -Version 0.28.0 introduces an `httpx.SSLContext()` class and `ssl_context` parameter. +This release introduces an `httpx.SSLContext()` class and `ssl_context` parameter. * Added `httpx.SSLContext` class and `ssl_context` parameter. (#3022, #3335) * The `verify` and `cert` arguments have been deprecated and will now raise warnings. (#3022, #3335) @@ -15,6 +15,7 @@ Version 0.28.0 introduces an `httpx.SSLContext()` class and `ssl_context` parame * The `URL.raw` property has now been removed. * Ensure JSON request bodies are compact. (#3363) * Review URL percent escape sets, based on WHATWG spec. (#3371, #3373) +* Ensure `certifi` and `httpcore` are only imported if required. (#3377) ## 0.27.2 (27th August, 2024) diff --git a/httpx/__version__.py b/httpx/__version__.py index 0a684ac3..5eaaddba 100644 --- a/httpx/__version__.py +++ b/httpx/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx" __description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.28.0" +__version__ = "0.27.2" diff --git a/httpx/_config.py b/httpx/_config.py index 2c9634a6..5a1a98a0 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -6,8 +6,6 @@ import sys import typing import warnings -import certifi - from ._models import Headers from ._types import HeaderTypes, TimeoutTypes from ._urls import URL @@ -77,6 +75,8 @@ class SSLContext(ssl.SSLContext): self, verify: bool = True, ) -> None: + import certifi + # ssl.SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. (from `ssl.create_default_context`) diff --git a/httpx/_main.py b/httpx/_main.py index 41c50f74..3df37cf0 100644 --- a/httpx/_main.py +++ b/httpx/_main.py @@ -6,7 +6,6 @@ import sys import typing import click -import httpcore import pygments.lexers import pygments.util import rich.console @@ -21,6 +20,9 @@ from ._exceptions import RequestError from ._models import Response from ._status_codes import codes +if typing.TYPE_CHECKING: + import httpcore # pragma: no cover + def print_help() -> None: console = rich.console.Console() diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index a1978c5a..50ff9105 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -27,11 +27,13 @@ client = httpx.Client(transport=transport) from __future__ import annotations import contextlib -import ssl import typing from types import TracebackType -import httpcore +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + import httpx # pragma: no cover from .._config import DEFAULT_LIMITS, Limits, Proxy, SSLContext, create_ssl_context from .._exceptions import ( @@ -66,9 +68,35 @@ SOCKET_OPTION = typing.Union[ __all__ = ["AsyncHTTPTransport", "HTTPTransport"] +HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {} + + +def _load_httpcore_exceptions() -> dict[type[Exception], type[httpx.HTTPError]]: + import httpcore + + return { + httpcore.TimeoutException: TimeoutException, + httpcore.ConnectTimeout: ConnectTimeout, + httpcore.ReadTimeout: ReadTimeout, + httpcore.WriteTimeout: WriteTimeout, + httpcore.PoolTimeout: PoolTimeout, + httpcore.NetworkError: NetworkError, + httpcore.ConnectError: ConnectError, + httpcore.ReadError: ReadError, + httpcore.WriteError: WriteError, + httpcore.ProxyError: ProxyError, + httpcore.UnsupportedProtocol: UnsupportedProtocol, + httpcore.ProtocolError: ProtocolError, + httpcore.LocalProtocolError: LocalProtocolError, + httpcore.RemoteProtocolError: RemoteProtocolError, + } + @contextlib.contextmanager def map_httpcore_exceptions() -> typing.Iterator[None]: + global HTTPCORE_EXC_MAP + if len(HTTPCORE_EXC_MAP) == 0: + HTTPCORE_EXC_MAP = _load_httpcore_exceptions() try: yield except Exception as exc: @@ -90,24 +118,6 @@ def map_httpcore_exceptions() -> typing.Iterator[None]: raise mapped_exc(message) from exc -HTTPCORE_EXC_MAP = { - httpcore.TimeoutException: TimeoutException, - httpcore.ConnectTimeout: ConnectTimeout, - httpcore.ReadTimeout: ReadTimeout, - httpcore.WriteTimeout: WriteTimeout, - httpcore.PoolTimeout: PoolTimeout, - httpcore.NetworkError: NetworkError, - httpcore.ConnectError: ConnectError, - httpcore.ReadError: ReadError, - httpcore.WriteError: WriteError, - httpcore.ProxyError: ProxyError, - httpcore.UnsupportedProtocol: UnsupportedProtocol, - httpcore.ProtocolError: ProtocolError, - httpcore.LocalProtocolError: LocalProtocolError, - httpcore.RemoteProtocolError: RemoteProtocolError, -} - - class ResponseStream(SyncByteStream): def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None: self._httpcore_stream = httpcore_stream @@ -138,6 +148,8 @@ class HTTPTransport(BaseTransport): verify: typing.Any = None, cert: typing.Any = None, ) -> None: + import httpcore + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if verify is not None or cert is not None: # pragma: nocover # Deprecated... @@ -225,6 +237,7 @@ class HTTPTransport(BaseTransport): request: Request, ) -> Response: assert isinstance(request.stream, SyncByteStream) + import httpcore req = httpcore.Request( method=request.method, @@ -284,6 +297,8 @@ class AsyncHTTPTransport(AsyncBaseTransport): verify: typing.Any = None, cert: typing.Any = None, ) -> None: + import httpcore + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy if verify is not None or cert is not None: # pragma: nocover # Deprecated... @@ -371,6 +386,7 @@ class AsyncHTTPTransport(AsyncBaseTransport): request: Request, ) -> Response: assert isinstance(request.stream, AsyncByteStream) + import httpcore req = httpcore.Request( method=request.method, diff --git a/tests/test_api.py b/tests/test_api.py index fe8083fc..225f384e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -85,3 +85,18 @@ def test_stream(server): def test_get_invalid_url(): with pytest.raises(httpx.UnsupportedProtocol): httpx.get("invalid://example.org") + + +# check that httpcore isn't imported until we do a request +def test_httpcore_lazy_loading(server): + import sys + + # unload our module if it is already loaded + if "httpx" in sys.modules: + del sys.modules["httpx"] + del sys.modules["httpcore"] + import httpx + + assert "httpcore" not in sys.modules + _response = httpx.get(server.url) + assert "httpcore" in sys.modules diff --git a/tests/test_config.py b/tests/test_config.py index 9f86f839..5d8748d1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -188,3 +188,18 @@ def test_proxy_with_auth_from_url(): def test_invalid_proxy_scheme(): with pytest.raises(ValueError): httpx.Proxy("invalid://example.com") + + +def test_certifi_lazy_loading(): + global httpx, certifi + import sys + + del sys.modules["httpx"] + del sys.modules["certifi"] + del httpx + del certifi + import httpx + + assert "certifi" not in sys.modules + _context = httpx.SSLContext() + assert "certifi" in sys.modules From 12be5c44caa7a34bfe19f82eace5150ddeba2612 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 29 Oct 2024 22:10:33 +0800 Subject: [PATCH 4/4] add socks5h proxy support (#3178) Signed-off-by: bin liu Co-authored-by: Tom Christie --- CHANGELOG.md | 1 + httpx/_config.py | 2 +- httpx/_transports/default.py | 8 ++++---- tests/client/test_proxies.py | 17 +++++++++-------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d32e53d..460e3154 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This release introduces an `httpx.SSLContext()` class and `ssl_context` paramete * Ensure JSON request bodies are compact. (#3363) * Review URL percent escape sets, based on WHATWG spec. (#3371, #3373) * Ensure `certifi` and `httpcore` are only imported if required. (#3377) +* Treat `socks5h` as a valid proxy scheme. (#3178) ## 0.27.2 (27th August, 2024) diff --git a/httpx/_config.py b/httpx/_config.py index 5a1a98a0..3fd5e1dd 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -262,7 +262,7 @@ class Proxy: url = URL(url) headers = Headers(headers) - if url.scheme not in ("http", "https", "socks5"): + if url.scheme not in ("http", "https", "socks5", "socks5h"): raise ValueError(f"Unknown scheme for proxy URL {url!r}") if url.username or url.password: diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 50ff9105..85d0f5f5 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -189,7 +189,7 @@ class HTTPTransport(BaseTransport): http2=http2, socket_options=socket_options, ) - elif proxy.url.scheme == "socks5": + elif proxy.url.scheme in ("socks5", "socks5h"): try: import socksio # noqa except ImportError: # pragma: no cover @@ -215,7 +215,7 @@ class HTTPTransport(BaseTransport): ) else: # pragma: no cover raise ValueError( - "Proxy protocol must be either 'http', 'https', or 'socks5'," + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," f" but got {proxy.url.scheme!r}." ) @@ -338,7 +338,7 @@ class AsyncHTTPTransport(AsyncBaseTransport): http2=http2, socket_options=socket_options, ) - elif proxy.url.scheme == "socks5": + elif proxy.url.scheme in ("socks5", "socks5h"): try: import socksio # noqa except ImportError: # pragma: no cover @@ -364,7 +364,7 @@ class AsyncHTTPTransport(AsyncBaseTransport): ) else: # pragma: no cover raise ValueError( - "Proxy protocol must be either 'http', 'https', or 'socks5'," + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," " but got {proxy.url.scheme!r}." ) diff --git a/tests/client/test_proxies.py b/tests/client/test_proxies.py index 90a92f56..3e4090dc 100644 --- a/tests/client/test_proxies.py +++ b/tests/client/test_proxies.py @@ -16,15 +16,16 @@ def url_to_origin(url: str) -> httpcore.URL: def test_socks_proxy(): url = httpx.URL("http://www.example.com") - client = httpx.Client(proxy="socks5://localhost/") - transport = client._transport_for_url(url) - assert isinstance(transport, httpx.HTTPTransport) - assert isinstance(transport._pool, httpcore.SOCKSProxy) + for proxy in ("socks5://localhost/", "socks5h://localhost/"): + client = httpx.Client(proxy=proxy) + transport = client._transport_for_url(url) + assert isinstance(transport, httpx.HTTPTransport) + assert isinstance(transport._pool, httpcore.SOCKSProxy) - async_client = httpx.AsyncClient(proxy="socks5://localhost/") - async_transport = async_client._transport_for_url(url) - assert isinstance(async_transport, httpx.AsyncHTTPTransport) - assert isinstance(async_transport._pool, httpcore.AsyncSOCKSProxy) + async_client = httpx.AsyncClient(proxy=proxy) + async_transport = async_client._transport_for_url(url) + assert isinstance(async_transport, httpx.AsyncHTTPTransport) + assert isinstance(async_transport._pool, httpcore.AsyncSOCKSProxy) PROXY_URL = "http://[::1]"