From 3faa4a8f2e0d406167b913bbfd3afab087662d03 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 14 Feb 2024 11:14:02 +0000 Subject: [PATCH 01/21] Improve 'Custom transports' docs (#3081) --- docs/advanced/transports.md | 85 ++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/docs/advanced/transports.md b/docs/advanced/transports.md index 7e0e21c6..d4e7615d 100644 --- a/docs/advanced/transports.md +++ b/docs/advanced/transports.md @@ -2,7 +2,7 @@ HTTPX's `Client` also accepts a `transport` argument. This argument allows you to provide a custom Transport object that will be used to perform the actual sending of the requests. -## HTTPTransport +## HTTP Transport For some advanced configuration you might need to instantiate a transport class directly, and pass it to the client instance. One example is the @@ -83,7 +83,7 @@ with httpx.Client(transport=transport, base_url="http://testserver") as client: ... ``` -## ASGITransport +## ASGI Transport You can configure an `httpx` client to call directly into an async Python web application using the ASGI protocol. @@ -148,7 +148,7 @@ However it is suggested to use `LifespanManager` from [asgi-lifespan](https://gi ## Custom transports -A transport instance must implement the low-level Transport API, which deals +A transport instance must implement the low-level Transport API which deals with sending a single request, and returning a response. You should either subclass `httpx.BaseTransport` to implement a transport to use with `Client`, or subclass `httpx.AsyncBaseTransport` to implement a transport to @@ -166,28 +166,81 @@ A complete example of a custom transport implementation would be: import json import httpx - class HelloWorldTransport(httpx.BaseTransport): """ A mock transport that always returns a JSON "Hello, world!" response. """ def handle_request(self, request): - message = {"text": "Hello, world!"} - content = json.dumps(message).encode("utf-8") - stream = httpx.ByteStream(content) - headers = [(b"content-type", b"application/json")] - return httpx.Response(200, headers=headers, stream=stream) + return httpx.Response(200, json={"text": "Hello, world!"}) ``` -Which we can use in the same way: +Or this example, which uses a custom transport and `httpx.Mounts` to always redirect `http://` requests. -```pycon ->>> import httpx ->>> client = httpx.Client(transport=HelloWorldTransport()) ->>> response = client.get("https://example.org/") ->>> response.json() -{"text": "Hello, world!"} +```python +class HTTPSRedirect(httpx.BaseTransport): + """ + A transport that always redirects to HTTPS. + """ + def handle_request(self, request): + url = request.url.copy_with(scheme="https") + return httpx.Response(303, headers={"Location": str(url)}) + +# A client where any `http` requests are always redirected to `https` +transport = httpx.Mounts({ + 'http://': HTTPSRedirect() + 'https://': httpx.HTTPTransport() +}) +client = httpx.Client(transport=transport) +``` + +A useful pattern here is custom transport classes that wrap the default HTTP implementation. For example... + +```python +class DebuggingTransport(httpx.BaseTransport): + def __init__(self, **kwargs): + self._wrapper = httpx.HTTPTransport(**kwargs) + + def handle_request(self, request): + print(f">>> {request}") + response = self._wrapper.handle_request(request) + print(f"<<< {response}") + return response + + def close(self): + self._wrapper.close() + +transport = DebuggingTransport() +client = httpx.Client(transport=transport) +``` + +Here's another case, where we're using a round-robin across a number of different proxies... + +```python +class ProxyRoundRobin(httpx.BaseTransport): + def __init__(self, proxies, **kwargs): + self._transports = [ + httpx.HTTPTransport(proxy=proxy, **kwargs) + for proxy in proxies + ] + self._idx = 0 + + def handle_request(self, request): + transport = self._transports[self._idx] + self._idx = (self._idx + 1) % len(self._transports) + return transport.handle_request(request) + + def close(self): + for transport in self._transports: + transport.close() + +proxies = [ + httpx.Proxy("http://127.0.0.1:8081"), + httpx.Proxy("http://127.0.0.1:8082"), + httpx.Proxy("http://127.0.0.1:8083"), +] +transport = ProxyRoundRobin(proxies=proxies) +client = httpx.Client(transport=transport) ``` ## Mock transports From 326b9431c761e1ef1e00b9f760d1f654c8db48c6 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 21 Feb 2024 13:06:19 +0000 Subject: [PATCH 02/21] Version 0.27.0 (#3095) * Version 0.27.0 * Update CHANGELOG.md (#3097) wrong year I think? I'm new to github so idk if I'm doing this right Co-authored-by: ReadyRainFor <119354484+ReadyRainFor@users.noreply.github.com> * Update CHANGELOG.md * Update CHANGELOG.md --------- Co-authored-by: Rain <119354484+Rainkenstein@users.noreply.github.com> Co-authored-by: ReadyRainFor <119354484+ReadyRainFor@users.noreply.github.com> --- CHANGELOG.md | 2 ++ httpx/__version__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7950a5f3..c063c081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +## 0.27.0 (21st February, 2024) + ### Deprecated * The `app=...` shortcut has been deprecated. Use the explicit style of `transport=httpx.WSGITransport()` or `transport=httpx.ASGITransport()` instead. diff --git a/httpx/__version__.py b/httpx/__version__.py index 3edc842c..c121a898 100644 --- a/httpx/__version__.py +++ b/httpx/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx" __description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.26.0" +__version__ = "0.27.0" From 77cb36f181b6ee728dd25c781ba5550a8804ee1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:52:20 +0000 Subject: [PATCH 03/21] Bump cryptography from 42.0.2 to 42.0.4 (#3107) Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f127064b..cc72ea12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ twine==4.0.2 # Tests & Linting coverage[toml]==7.4.1 -cryptography==42.0.2 +cryptography==42.0.4 mypy==1.8.0 pytest==8.0.0 ruff==0.1.15 From 87713d2172053c4ad05efacf3ab7e0a5c15616fc Mon Sep 17 00:00:00 2001 From: Kar Petrosyan <92274156+karpetrosyan@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:30:05 +0400 Subject: [PATCH 04/21] Define and expose the API from the same place (#3106) * Tidy up imports * Update tests/test_exported_members.py --------- Co-authored-by: Tom Christie --- httpx/__init__.py | 55 +++++++---------------------------- httpx/_api.py | 12 ++++++++ httpx/_auth.py | 3 ++ httpx/_client.py | 2 ++ httpx/_config.py | 2 ++ httpx/_content.py | 2 ++ httpx/_exceptions.py | 31 ++++++++++++++++++++ httpx/_models.py | 2 ++ httpx/_status_codes.py | 2 ++ httpx/_transports/__init__.py | 15 ++++++++++ httpx/_transports/asgi.py | 2 ++ httpx/_transports/base.py | 2 ++ httpx/_transports/default.py | 2 ++ httpx/_transports/mock.py | 3 ++ httpx/_transports/wsgi.py | 3 ++ httpx/_types.py | 2 ++ httpx/_urls.py | 2 ++ pyproject.toml | 3 ++ 18 files changed, 101 insertions(+), 44 deletions(-) diff --git a/httpx/__init__.py b/httpx/__init__.py index f61112f8..e9addde0 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -1,48 +1,15 @@ from .__version__ import __description__, __title__, __version__ -from ._api import delete, get, head, options, patch, post, put, request, stream -from ._auth import Auth, BasicAuth, DigestAuth, NetRCAuth -from ._client import USE_CLIENT_DEFAULT, AsyncClient, Client -from ._config import Limits, Proxy, Timeout, create_ssl_context -from ._content import ByteStream -from ._exceptions import ( - CloseError, - ConnectError, - ConnectTimeout, - CookieConflict, - DecodingError, - HTTPError, - HTTPStatusError, - InvalidURL, - LocalProtocolError, - NetworkError, - PoolTimeout, - ProtocolError, - ProxyError, - ReadError, - ReadTimeout, - RemoteProtocolError, - RequestError, - RequestNotRead, - ResponseNotRead, - StreamClosed, - StreamConsumed, - StreamError, - TimeoutException, - TooManyRedirects, - TransportError, - UnsupportedProtocol, - WriteError, - WriteTimeout, -) -from ._models import Cookies, Headers, Request, Response -from ._status_codes import codes -from ._transports.asgi import ASGITransport -from ._transports.base import AsyncBaseTransport, BaseTransport -from ._transports.default import AsyncHTTPTransport, HTTPTransport -from ._transports.mock import MockTransport -from ._transports.wsgi import WSGITransport -from ._types import AsyncByteStream, SyncByteStream -from ._urls import URL, QueryParams +from ._api import * +from ._auth import * +from ._client import * +from ._config import * +from ._content import * +from ._exceptions import * +from ._models import * +from ._status_codes import * +from ._transports import * +from ._types import * +from ._urls import * try: from ._main import main diff --git a/httpx/_api.py b/httpx/_api.py index b5821cc4..3dd943b3 100644 --- a/httpx/_api.py +++ b/httpx/_api.py @@ -22,6 +22,18 @@ from ._types import ( VerifyTypes, ) +__all__ = [ + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "request", + "stream", +] + def request( method: str, diff --git a/httpx/_auth.py b/httpx/_auth.py index 903e3996..b03971ab 100644 --- a/httpx/_auth.py +++ b/httpx/_auth.py @@ -16,6 +16,9 @@ if typing.TYPE_CHECKING: # pragma: no cover from hashlib import _Hash +__all__ = ["Auth", "BasicAuth", "DigestAuth", "NetRCAuth"] + + class Auth: """ Base class for all authentication schemes. diff --git a/httpx/_client.py b/httpx/_client.py index e2c6702e..cf3b3062 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -58,6 +58,8 @@ from ._utils import ( same_origin, ) +__all__ = ["USE_CLIENT_DEFAULT", "AsyncClient", "Client"] + # The type annotation for @classmethod and context managers here follows PEP 484 # https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods T = typing.TypeVar("T", bound="Client") diff --git a/httpx/_config.py b/httpx/_config.py index 7636a5dc..6662ea80 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -14,6 +14,8 @@ from ._types import CertTypes, HeaderTypes, TimeoutTypes, URLTypes, VerifyTypes from ._urls import URL from ._utils import get_ca_bundle_from_env +__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] + DEFAULT_CIPHERS = ":".join( [ "ECDHE+AESGCM", diff --git a/httpx/_content.py b/httpx/_content.py index 10b574bb..786699f3 100644 --- a/httpx/_content.py +++ b/httpx/_content.py @@ -25,6 +25,8 @@ from ._types import ( ) from ._utils import peek_filelike_length, primitive_value_to_str +__all__ = ["ByteStream"] + class ByteStream(AsyncByteStream, SyncByteStream): def __init__(self, stream: bytes) -> None: diff --git a/httpx/_exceptions.py b/httpx/_exceptions.py index 11424621..18dfa2f2 100644 --- a/httpx/_exceptions.py +++ b/httpx/_exceptions.py @@ -38,6 +38,37 @@ import typing if typing.TYPE_CHECKING: from ._models import Request, Response # pragma: no cover +__all__ = [ + "CloseError", + "ConnectError", + "ConnectTimeout", + "CookieConflict", + "DecodingError", + "HTTPError", + "HTTPStatusError", + "InvalidURL", + "LocalProtocolError", + "NetworkError", + "PoolTimeout", + "ProtocolError", + "ProxyError", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "RequestError", + "RequestNotRead", + "ResponseNotRead", + "StreamClosed", + "StreamConsumed", + "StreamError", + "TimeoutException", + "TooManyRedirects", + "TransportError", + "UnsupportedProtocol", + "WriteError", + "WriteTimeout", +] + class HTTPError(Exception): """ diff --git a/httpx/_models.py b/httpx/_models.py index cd76705f..92b393a2 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -53,6 +53,8 @@ from ._utils import ( parse_header_links, ) +__all__ = ["Cookies", "Headers", "Request", "Response"] + class Headers(typing.MutableMapping[str, str]): """ diff --git a/httpx/_status_codes.py b/httpx/_status_codes.py index 4cde4e68..133a6231 100644 --- a/httpx/_status_codes.py +++ b/httpx/_status_codes.py @@ -2,6 +2,8 @@ from __future__ import annotations from enum import IntEnum +__all__ = ["codes"] + class codes(IntEnum): """HTTP status codes and reason phrases diff --git a/httpx/_transports/__init__.py b/httpx/_transports/__init__.py index e69de29b..7a321053 100644 --- a/httpx/_transports/__init__.py +++ b/httpx/_transports/__init__.py @@ -0,0 +1,15 @@ +from .asgi import * +from .base import * +from .default import * +from .mock import * +from .wsgi import * + +__all__ = [ + "ASGITransport", + "AsyncBaseTransport", + "BaseTransport", + "AsyncHTTPTransport", + "HTTPTransport", + "MockTransport", + "WSGITransport", +] diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index 9543a128..794cb17b 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -25,6 +25,8 @@ _ASGIApp = typing.Callable[ [typing.Dict[str, typing.Any], _Receive, _Send], typing.Coroutine[None, None, None] ] +__all__ = ["ASGITransport"] + def create_event() -> Event: if sniffio.current_async_library() == "trio": diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index 8b6dc3c2..66fd99d7 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -8,6 +8,8 @@ from .._models import Request, Response T = typing.TypeVar("T", bound="BaseTransport") A = typing.TypeVar("A", bound="AsyncBaseTransport") +__all__ = ["AsyncBaseTransport", "BaseTransport"] + class BaseTransport: def __enter__(self: T) -> T: diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 14476a3c..e82104e9 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -62,6 +62,8 @@ SOCKET_OPTION = typing.Union[ typing.Tuple[int, int, None, int], ] +__all__ = ["AsyncHTTPTransport", "HTTPTransport"] + @contextlib.contextmanager def map_httpcore_exceptions() -> typing.Iterator[None]: diff --git a/httpx/_transports/mock.py b/httpx/_transports/mock.py index 5abea837..8c418f59 100644 --- a/httpx/_transports/mock.py +++ b/httpx/_transports/mock.py @@ -9,6 +9,9 @@ SyncHandler = typing.Callable[[Request], Response] AsyncHandler = typing.Callable[[Request], typing.Coroutine[None, None, Response]] +__all__ = ["MockTransport"] + + class MockTransport(AsyncBaseTransport, BaseTransport): def __init__(self, handler: SyncHandler | AsyncHandler) -> None: self.handler = handler diff --git a/httpx/_transports/wsgi.py b/httpx/_transports/wsgi.py index cd03a941..8592ffe0 100644 --- a/httpx/_transports/wsgi.py +++ b/httpx/_transports/wsgi.py @@ -16,6 +16,9 @@ if typing.TYPE_CHECKING: _T = typing.TypeVar("_T") +__all__ = ["WSGITransport"] + + def _skip_leading_empty_chunks(body: typing.Iterable[_T]) -> typing.Iterable[_T]: body = iter(body) for chunk in body: diff --git a/httpx/_types.py b/httpx/_types.py index 649d101d..b7b0518c 100644 --- a/httpx/_types.py +++ b/httpx/_types.py @@ -108,6 +108,8 @@ RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]] RequestExtensions = MutableMapping[str, Any] +__all__ = ["AsyncByteStream", "SyncByteStream"] + class SyncByteStream: def __iter__(self) -> Iterator[bytes]: diff --git a/httpx/_urls.py b/httpx/_urls.py index 43dedd56..f9f68a99 100644 --- a/httpx/_urls.py +++ b/httpx/_urls.py @@ -9,6 +9,8 @@ from ._types import QueryParamTypes, RawURL, URLTypes from ._urlparse import urlencode, urlparse from ._utils import primitive_value_to_str +__all__ = ["URL", "QueryParams"] + class URL: """ diff --git a/pyproject.toml b/pyproject.toml index 4f7a848f..3fe24a14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,9 @@ ignore = ["B904", "B028"] [tool.ruff.isort] combine-as-imports = true +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F403", "F405"] + [tool.mypy] ignore_missing_imports = true strict = true From 4de13707ee0fbc6e3ea5b30c90cf64aae0af7fb2 Mon Sep 17 00:00:00 2001 From: Jon Finerty Date: Fri, 23 Feb 2024 13:36:45 +0000 Subject: [PATCH 05/21] Use more permissible types in ASGIApp (#3109) * Use the type.MutableMapping instead of Dict MutableMapping is a slightly more permissible type (allowing the previous Dict type) but matches up to Starlettes tpyes * Update CHANGELOG.md --------- Co-authored-by: Tom Christie --- CHANGELOG.md | 4 ++++ httpx/_transports/asgi.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c063c081..2d8634f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Fixed + +* Fix `app` type signature in `ASGITransport`. (#3109) + ## 0.27.0 (21st February, 2024) ### Deprecated diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index 794cb17b..d1828f25 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -16,13 +16,13 @@ if typing.TYPE_CHECKING: # pragma: no cover Event = typing.Union[asyncio.Event, trio.Event] -_Message = typing.Dict[str, typing.Any] +_Message = typing.MutableMapping[str, typing.Any] _Receive = typing.Callable[[], typing.Awaitable[_Message]] _Send = typing.Callable[ - [typing.Dict[str, typing.Any]], typing.Coroutine[None, None, None] + [typing.MutableMapping[str, typing.Any]], typing.Awaitable[None] ] _ASGIApp = typing.Callable[ - [typing.Dict[str, typing.Any], _Receive, _Send], typing.Coroutine[None, None, None] + [typing.MutableMapping[str, typing.Any], _Receive, _Send], typing.Awaitable[None] ] __all__ = ["ASGITransport"] @@ -141,7 +141,7 @@ class ASGITransport(AsyncBaseTransport): return {"type": "http.request", "body": b"", "more_body": False} return {"type": "http.request", "body": body, "more_body": True} - async def send(message: dict[str, typing.Any]) -> None: + async def send(message: typing.MutableMapping[str, typing.Any]) -> None: nonlocal status_code, response_headers, response_started if message["type"] == "http.response.start": From e745060c75d06a7a6c1c90007120af2a43b3fffc Mon Sep 17 00:00:00 2001 From: T-256 <132141463+T-256@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:41:43 +0330 Subject: [PATCH 06/21] test `is_https_redirect` via public api (#3064) * test `is_https_redirect` via public api * Update tests/test_utils.py --- tests/test_utils.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0ef87d18..2341a7c7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,7 +11,6 @@ from httpx._utils import ( URLPattern, get_ca_bundle_from_env, get_environment_proxies, - is_https_redirect, same_origin, ) @@ -237,21 +236,39 @@ def test_not_same_origin(): def test_is_https_redirect(): - url = httpx.URL("http://example.com") - location = httpx.URL("https://example.com") - assert is_https_redirect(url, location) + url = httpx.URL("https://example.com") + request = httpx.Request( + "GET", "http://example.com", headers={"Authorization": "empty"} + ) + + client = httpx.Client() + headers = client._redirect_headers(request, url, "GET") + + assert "Authorization" in headers def test_is_not_https_redirect(): - url = httpx.URL("http://example.com") - location = httpx.URL("https://www.example.com") - assert not is_https_redirect(url, location) + url = httpx.URL("https://www.example.com") + request = httpx.Request( + "GET", "http://example.com", headers={"Authorization": "empty"} + ) + + client = httpx.Client() + headers = client._redirect_headers(request, url, "GET") + + assert "Authorization" not in headers def test_is_not_https_redirect_if_not_default_ports(): - url = httpx.URL("http://example.com:9999") - location = httpx.URL("https://example.com:1337") - assert not is_https_redirect(url, location) + url = httpx.URL("https://example.com:1337") + request = httpx.Request( + "GET", "http://example.com:9999", headers={"Authorization": "empty"} + ) + + client = httpx.Client() + headers = client._redirect_headers(request, url, "GET") + + assert "Authorization" not in headers @pytest.mark.parametrize( From fc84f7f6eb77fe5d4428261c837ac8016ec77a28 Mon Sep 17 00:00:00 2001 From: T-256 <132141463+T-256@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:46:03 +0330 Subject: [PATCH 07/21] test `same_origin` via public api (#3062) Co-authored-by: Tom Christie --- tests/test_utils.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2341a7c7..f98a18f2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,7 +11,6 @@ from httpx._utils import ( URLPattern, get_ca_bundle_from_env, get_environment_proxies, - same_origin, ) from .common import TESTS_DIR @@ -224,15 +223,23 @@ def test_obfuscate_sensitive_headers(headers, output): def test_same_origin(): - origin1 = httpx.URL("https://example.com") - origin2 = httpx.URL("HTTPS://EXAMPLE.COM:443") - assert same_origin(origin1, origin2) + origin = httpx.URL("https://example.com") + request = httpx.Request("GET", "HTTPS://EXAMPLE.COM:443") + + client = httpx.Client() + headers = client._redirect_headers(request, origin, "GET") + + assert headers["Host"] == request.url.netloc.decode("ascii") def test_not_same_origin(): - origin1 = httpx.URL("https://example.com") - origin2 = httpx.URL("HTTP://EXAMPLE.COM") - assert not same_origin(origin1, origin2) + origin = httpx.URL("https://example.com") + request = httpx.Request("GET", "HTTP://EXAMPLE.COM:80") + + client = httpx.Client() + headers = client._redirect_headers(request, origin, "GET") + + assert headers["Host"] == origin.netloc.decode("ascii") def test_is_https_redirect(): From df5345140e09ac6c2de0d9589bcd6f3e31c6aa3f Mon Sep 17 00:00:00 2001 From: akgnah Date: Fri, 23 Feb 2024 22:33:15 +0800 Subject: [PATCH 08/21] fix docs basic authentication typo (#3112) Signed-off-by: akgnah <1024@setq.me> Co-authored-by: Tom Christie --- docs/advanced/authentication.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/advanced/authentication.md b/docs/advanced/authentication.md index edcc15f8..63d26e5f 100644 --- a/docs/advanced/authentication.md +++ b/docs/advanced/authentication.md @@ -1,7 +1,7 @@ Authentication can either be included on a per-request basis... ```pycon ->>> auth = httpx.BasicAuthentication(username="username", password="secret") +>>> auth = httpx.BasicAuth(username="username", password="secret") >>> client = httpx.Client() >>> response = client.get("https://www.example.com/", auth=auth) ``` @@ -9,7 +9,7 @@ Authentication can either be included on a per-request basis... Or configured on the client instance, ensuring that all outgoing requests will include authentication credentials... ```pycon ->>> auth = httpx.BasicAuthentication(username="username", password="secret") +>>> auth = httpx.BasicAuth(username="username", password="secret") >>> client = httpx.Client(auth=auth) >>> response = client.get("https://www.example.com/") ``` @@ -19,7 +19,7 @@ Or configured on the client instance, ensuring that all outgoing requests will i HTTP basic authentication is an unencrypted authentication scheme that uses a simple encoding of the username and password in the request `Authorization` header. Since it is unencrypted it should typically only be used over `https`, although this is not strictly enforced. ```pycon ->>> auth = httpx.BasicAuthentication(username="finley", password="secret") +>>> auth = httpx.BasicAuth(username="finley", password="secret") >>> client = httpx.Client(auth=auth) >>> response = client.get("https://httpbin.org/basic-auth/finley/secret") >>> response From 6d852d319acd5d97caf14037dff15ede04b37542 Mon Sep 17 00:00:00 2001 From: Alex <8340441+alexgmin@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:36:58 +0100 Subject: [PATCH 09/21] Fix client.send() timeout new Request instance (#3116) --- httpx/_client.py | 13 +++++++++++++ tests/test_timeouts.py | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/httpx/_client.py b/httpx/_client.py index cf3b3062..ba5decf8 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -562,6 +562,15 @@ class BaseClient: return request.stream + def _set_timeout(self, request: Request) -> None: + if "timeout" not in request.extensions: + timeout = ( + self.timeout + if isinstance(self.timeout, UseClientDefault) + else Timeout(self.timeout) + ) + request.extensions = dict(**request.extensions, timeout=timeout.as_dict()) + class Client(BaseClient): """ @@ -911,6 +920,8 @@ class Client(BaseClient): else follow_redirects ) + self._set_timeout(request) + auth = self._build_request_auth(request, auth) response = self._send_handling_auth( @@ -1658,6 +1669,8 @@ class AsyncClient(BaseClient): else follow_redirects ) + self._set_timeout(request) + auth = self._build_request_auth(request, auth) response = await self._send_handling_auth( diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py index 09b25160..666cc8e3 100644 --- a/tests/test_timeouts.py +++ b/tests/test_timeouts.py @@ -42,3 +42,14 @@ async def test_pool_timeout(server): with pytest.raises(httpx.PoolTimeout): async with client.stream("GET", server.url): await client.get(server.url) + + +@pytest.mark.anyio +async def test_async_client_new_request_send_timeout(server): + timeout = httpx.Timeout(1e-6) + + async with httpx.AsyncClient(timeout=timeout) as client: + with pytest.raises(httpx.TimeoutException): + await client.send( + httpx.Request("GET", server.url.copy_with(path="/slow_response")) + ) From 6045186f7d3a665b965c13dec413e3a346b1a01d Mon Sep 17 00:00:00 2001 From: Nick Cameron <146452552+nick2i@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:13:23 -0500 Subject: [PATCH 10/21] Update /advanced/# links -> /advanced/clients/# (#3123) --- httpx/_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index ba5decf8..4645308f 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -342,7 +342,7 @@ class BaseClient: See also: [Request instances][0] - [0]: /advanced/#request-instances + [0]: /advanced/clients/#request-instances """ url = self._merge_url(url) headers = self._merge_headers(headers) @@ -812,7 +812,7 @@ class Client(BaseClient): [Merging of configuration][0] for how the various parameters are merged with client-level configuration. - [0]: /advanced/#merging-of-configuration + [0]: /advanced/clients/#merging-of-configuration """ if cookies is not None: message = ( @@ -908,7 +908,7 @@ class Client(BaseClient): See also: [Request instances][0] - [0]: /advanced/#request-instances + [0]: /advanced/clients/#request-instances """ if self._state == ClientState.CLOSED: raise RuntimeError("Cannot send a request, as the client has been closed.") @@ -1560,7 +1560,7 @@ class AsyncClient(BaseClient): and [Merging of configuration][0] for how the various parameters are merged with client-level configuration. - [0]: /advanced/#merging-of-configuration + [0]: /advanced/clients/#merging-of-configuration """ if cookies is not None: # pragma: no cover @@ -1657,7 +1657,7 @@ class AsyncClient(BaseClient): See also: [Request instances][0] - [0]: /advanced/#request-instances + [0]: /advanced/clients/#request-instances """ if self._state == ClientState.CLOSED: raise RuntimeError("Cannot send a request, as the client has been closed.") From 4941b40cbbae1ceb2b7a3d4e8064df96806c5e33 Mon Sep 17 00:00:00 2001 From: Nick Cameron <146452552+nick2i@users.noreply.github.com> Date: Thu, 29 Feb 2024 06:11:43 -0500 Subject: [PATCH 11/21] Fix broken links in docs/contributing.md and CHANGELOG.md (#3124) --- CHANGELOG.md | 10 +++++----- docs/contributing.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8634f0..85d3bcec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,7 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). * The logging behaviour has been changed to be more in-line with other standard Python logging usages. We no longer have a custom `TRACE` log level, and we no longer use the `HTTPX_LOG_LEVEL` environment variable to auto-configure logging. We now have a significant amount of `DEBUG` logging available at the network level. Full documentation is available at https://www.python-httpx.org/logging/ (#2547, encode/httpcore#648) * The `Response.iter_lines()` method now matches the stdlib behaviour and does not include the newline characters. It also resolves a performance issue. (#2423) * Query parameter encoding switches from using + for spaces and %2F for forward slash, to instead using %20 for spaces and treating forward slash as a safe, unescaped character. This differs from `requests`, but is in line with browser behavior in Chrome, Safari, and Firefox. Both options are RFC valid. (#2543) -* NetRC authentication is no longer automatically handled, but is instead supported by an explicit `httpx.NetRCAuth()` authentication class. See the documentation at https://www.python-httpx.org/advanced/#netrc-support (#2525) +* NetRC authentication is no longer automatically handled, but is instead supported by an explicit `httpx.NetRCAuth()` authentication class. See the documentation at https://www.python-httpx.org/advanced/authentication/#netrc-authentication (#2525) ### Removed @@ -151,7 +151,7 @@ See the "Removed" section of these release notes for details. ### Changed * Drop support for Python 3.6. (#2097) -* Use `utf-8` as the default character set, instead of falling back to `charset-normalizer` for auto-detection. To enable automatic character set detection, see [the documentation](https://www.python-httpx.org/advanced/#character-set-encodings-and-auto-detection). (#2165) +* Use `utf-8` as the default character set, instead of falling back to `charset-normalizer` for auto-detection. To enable automatic character set detection, see [the documentation](https://www.python-httpx.org/advanced/text-encodings/#using-auto-detection). (#2165) ### Fixed @@ -170,7 +170,7 @@ See the "Removed" section of these release notes for details. ### Added -* Support for [the SOCKS5 proxy protocol](https://www.python-httpx.org/advanced/#socks) via [the `socksio` package](https://github.com/sethmlarson/socksio). (#2034) +* Support for [the SOCKS5 proxy protocol](https://www.python-httpx.org/advanced/proxies/#socks) via [the `socksio` package](https://github.com/sethmlarson/socksio). (#2034) * Support for custom headers in multipart/form-data requests (#1936) ### Fixed @@ -325,7 +325,7 @@ finally: The 0.18.x release series formalises our low-level Transport API, introducing the base classes `httpx.BaseTransport` and `httpx.AsyncBaseTransport`. -See the "[Writing custom transports](https://www.python-httpx.org/advanced/#writing-custom-transports)" documentation and the [`httpx.BaseTransport.handle_request()`](https://github.com/encode/httpx/blob/397aad98fdc8b7580a5fc3e88f1578b4302c6382/httpx/_transports/base.py#L77-L147) docstring for more complete details on implementing custom transports. +See the "[Custom transports](https://www.python-httpx.org/advanced/transports/#custom-transports)" documentation and the [`httpx.BaseTransport.handle_request()`](https://github.com/encode/httpx/blob/397aad98fdc8b7580a5fc3e88f1578b4302c6382/httpx/_transports/base.py#L77-L147) docstring for more complete details on implementing custom transports. Pull request #1522 includes a checklist of differences from the previous `httpcore` transport API, for developers implementing custom transports. @@ -642,7 +642,7 @@ This release switches to `httpcore` for all the internal networking, which means It also means we've had to remove our UDS support, since maintaining that would have meant having to push back our work towards a 1.0 release, which isn't a trade-off we wanted to make. -We also now have [a public "Transport API"](https://www.python-httpx.org/advanced/#custom-transports), which you can use to implement custom transport implementations against. This formalises and replaces our previously private "Dispatch API". +We also now have [a public "Transport API"](https://www.python-httpx.org/advanced/transports/#custom-transports), which you can use to implement custom transport implementations against. This formalises and replaces our previously private "Dispatch API". ### Changed diff --git a/docs/contributing.md b/docs/contributing.md index 47dd9dc5..0d3ad5f1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -206,8 +206,8 @@ UI options. At this point the server is ready to start serving requests, you'll need to configure HTTPX as described in the -[proxy section](https://www.python-httpx.org/advanced/#http-proxying) and -the [SSL certificates section](https://www.python-httpx.org/advanced/#ssl-certificates), +[proxy section](https://www.python-httpx.org/advanced/proxies/#http-proxies) and +the [SSL certificates section](https://www.python-httpx.org/advanced/ssl/), this is where our previously generated `client.pem` comes in: ``` From 7e10342c2ad44955582e4fef96470f3e653347d4 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 29 Feb 2024 11:42:17 +0000 Subject: [PATCH 12/21] Delete `README_chinese.md` (#3122) Discussed in https://github.com/encode/httpx/discussions/3024 Having translated versions for our users is friendly, but we're not doing this in a consistent way. --- README_chinese.md | 144 ---------------------------------------------- 1 file changed, 144 deletions(-) delete mode 100644 README_chinese.md diff --git a/README_chinese.md b/README_chinese.md deleted file mode 100644 index ad20c5a1..00000000 --- a/README_chinese.md +++ /dev/null @@ -1,144 +0,0 @@ -

- HTTPX -

- -

HTTPX - 适用于 Python 的下一代 HTTP 客户端

- -

- - Test Suite - - - Package version - -

- -HTTPX 是适用于 Python3 的功能齐全的 HTTP 客户端。 它集成了 **一个命令行客户端**,同时支持 **HTTP/1.1 和 HTTP/2**,并提供了 **同步和异步 API**。 - ---- - -通过 pip 安装 HTTPX: - -```shell -$ pip install httpx -``` - -使用 httpx: - -```pycon ->>> import httpx ->>> r = httpx.get('https://www.example.org/') ->>> r - ->>> r.status_code -200 ->>> r.headers['content-type'] -'text/html; charset=UTF-8' ->>> r.text -'\n\n\nExample Domain...' -``` - -或者使用命令行客户端。 - -```shell -$ pip install 'httpx[cli]' # 命令行功能是可选的。 -``` - -它允许我们直接通过命令行来使用 HTTPX... - -

- httpx --help -

- -发送一个请求... - -

- httpx http://httpbin.org/json -

- -## 特性 - -HTTPX 建立在成熟的 requests 可用性基础上,为您提供以下功能: - -* 广泛的 [requests 兼容 API](https://www.python-httpx.org/compatibility/)。 -* 内置的命令行客户端功能。 -* HTTP/1.1 [和 HTTP/2 支持](https://www.python-httpx.org/http2/)。 -* 标准同步接口,也支持 [异步](https://www.python-httpx.org/async/)。 -* 能够直接向 [WSGI 应用发送请求](https://www.python-httpx.org/advanced/#calling-into-python-web-apps) 或向 [ASGI 应用发送请求](https://www.python-httpx.org/async/#calling-into-python-web-apps)。 -* 每一处严格的超时控制。 -* 完整的类型注解。 -* 100% 测试。 - -加上这些应该具备的标准功能... - -* 国际化域名与 URL -* Keep-Alive & 连接池 -* Cookie 持久性会话 -* 浏览器风格的 SSL 验证 -* 基础或摘要身份验证 -* 优雅的键值 Cookies -* 自动解压缩 -* 内容自动解码 -* Unicode 响应正文 -* 分段文件上传 -* HTTP(S)代理支持 -* 可配置的连接超时 -* 流式下载 -* .netrc 支持 -* 分块请求 - -## 安装 - -使用 pip 安装: - -```shell -$ pip install httpx -``` - -或者,安装可选的 HTTP/2 支持: - -```shell -$ pip install httpx[http2] -``` - -HTTPX 要求 Python 3.8+ 版本。 - -## 文档 - -项目文档现已就绪,请访问 [https://www.python-httpx.org/](https://www.python-httpx.org/) 来阅读。 - -要浏览所有基础知识,请访问 [快速开始](https://www.python-httpx.org/quickstart/)。 - -更高级的主题,可参阅 [高级用法](https://www.python-httpx.org/advanced/) 章节, [异步支持](https://www.python-httpx.org/async/) 或者 [HTTP/2](https://www.python-httpx.org/http2/) 章节。 - -[Developer Interface](https://www.python-httpx.org/api/) 提供了全面的 API 参考。 - -要了解与 HTTPX 集成的工具, 请访问 [第三方包](https://www.python-httpx.org/third_party_packages/)。 - -## 贡献 - -如果您想对本项目做出贡献,请访问 [贡献者指南](https://www.python-httpx.org/contributing/) 来了解如何开始。 - -## 依赖 - -HTTPX 项目依赖于这些优秀的库: - -* `httpcore` - `httpx` 基础传输接口实现。 - * `h11` - HTTP/1.1 支持。 -* `certifi` - SSL 证书。 -* `idna` - 国际化域名支持。 -* `sniffio` - 异步库自动检测。 - -以及这些可选的安装: - -* `h2` - HTTP/2 支持。 *(可选的,通过 `httpx[http2]`)* -* `socksio` - SOCKS 代理支持。 *(可选的, 通过 `httpx[socks]`)* -* `rich` - 丰富的终端支持。 *(可选的,通过 `httpx[cli]`)* -* `click` - 命令行客户端支持。 *(可选的,通过 `httpx[cli]`)* -* `brotli` 或者 `brotlicffi` - 对 “brotli” 压缩响应的解码。*(可选的,通过 `httpx[brotli]`)* - -这项工作的大量功劳都归功于参考了 `requests` 所遵循的 API 结构,以及 `urllib3` 中众多围绕底层网络细节的设计灵感。 - ---- - -

HTTPX 使用 BSD 开源协议 code。
精心设计和制作。

— 🦋 —

From f3eb3c90fdd19d2e4c5239e19a3588d072ff53fb Mon Sep 17 00:00:00 2001 From: Kar Petrosyan <92274156+karpetrosyan@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:40:07 -0500 Subject: [PATCH 13/21] Keep clients in sync (#3120) Co-authored-by: Tom Christie --- httpx/_client.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index 4645308f..d95877e8 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -1054,7 +1054,7 @@ class Client(BaseClient): params: QueryParamTypes | None = None, headers: HeaderTypes | None = None, cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, extensions: RequestExtensions | None = None, @@ -1391,8 +1391,7 @@ class AsyncClient(BaseClient): follow_redirects: bool = False, limits: Limits = DEFAULT_LIMITS, max_redirects: int = DEFAULT_MAX_REDIRECTS, - event_hooks: None - | (typing.Mapping[str, list[typing.Callable[..., typing.Any]]]) = None, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, base_url: URLTypes = "", transport: AsyncBaseTransport | None = None, app: typing.Callable[..., typing.Any] | None = None, @@ -1438,7 +1437,7 @@ class AsyncClient(BaseClient): ) warnings.warn(message, DeprecationWarning) - allow_env_proxies = trust_env and transport is None + allow_env_proxies = trust_env and app is None and transport is None proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( @@ -1599,7 +1598,7 @@ class AsyncClient(BaseClient): params: QueryParamTypes | None = None, headers: HeaderTypes | None = None, cookies: CookieTypes | None = None, - auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, extensions: RequestExtensions | None = None, From 0006ed0547f8f8a3cbe2edf758e996c3c73b5e7d Mon Sep 17 00:00:00 2001 From: T-256 <132141463+T-256@users.noreply.github.com> Date: Fri, 1 Mar 2024 23:19:23 +0330 Subject: [PATCH 14/21] format (#3131) Co-authored-by: T-256 --- httpx/_compat.py | 1 + httpx/_decoders.py | 1 + httpx/_exceptions.py | 1 + httpx/_transports/default.py | 1 + httpx/_urlparse.py | 1 + pyproject.toml | 5 ++--- tests/client/test_auth.py | 1 + tests/test_auth.py | 1 + tests/test_multipart.py | 4 ++-- 9 files changed, 11 insertions(+), 5 deletions(-) diff --git a/httpx/_compat.py b/httpx/_compat.py index 493e6210..27ccc682 100644 --- a/httpx/_compat.py +++ b/httpx/_compat.py @@ -2,6 +2,7 @@ The _compat module is used for code which requires branching between different Python environments. It is excluded from the code coverage checks. """ + import ssl import sys diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 31c72c7f..f9d3adbb 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -3,6 +3,7 @@ Handlers for Content-Encoding. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding """ + from __future__ import annotations import codecs diff --git a/httpx/_exceptions.py b/httpx/_exceptions.py index 18dfa2f2..77f45a6d 100644 --- a/httpx/_exceptions.py +++ b/httpx/_exceptions.py @@ -30,6 +30,7 @@ Our exception hierarchy: x ResponseNotRead x RequestNotRead """ + from __future__ import annotations import contextlib diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index e82104e9..bcc8bf42 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -23,6 +23,7 @@ client = httpx.Client(transport=transport) transport = httpx.HTTPTransport(uds="socket.uds") client = httpx.Client(transport=transport) """ + from __future__ import annotations import contextlib diff --git a/httpx/_urlparse.py b/httpx/_urlparse.py index 6a4b55b3..232269ee 100644 --- a/httpx/_urlparse.py +++ b/httpx/_urlparse.py @@ -15,6 +15,7 @@ Previously we relied on the excellent `rfc3986` package to handle URL parsing an validation, but this module provides a simpler alternative, with less indirection required. """ + from __future__ import annotations import ipaddress diff --git a/pyproject.toml b/pyproject.toml index 3fe24a14..9e6464c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,12 +93,11 @@ text = "\n---\n\n[Full changelog](https://github.com/encode/httpx/blob/master/CH pattern = 'src="(docs/img/.*?)"' replacement = 'src="https://raw.githubusercontent.com/encode/httpx/master/\1"' -# https://beta.ruff.rs/docs/configuration/#using-rufftoml -[tool.ruff] +[tool.ruff.lint] select = ["E", "F", "I", "B", "PIE"] ignore = ["B904", "B028"] -[tool.ruff.isort] +[tool.ruff.lint.isort] combine-as-imports = true [tool.ruff.lint.per-file-ignores] diff --git a/tests/client/test_auth.py b/tests/client/test_auth.py index e6bac23d..5776fc33 100644 --- a/tests/client/test_auth.py +++ b/tests/client/test_auth.py @@ -3,6 +3,7 @@ Integration tests for authentication. Unit tests for auth classes also exist in tests/test_auth.py """ + import hashlib import netrc import os diff --git a/tests/test_auth.py b/tests/test_auth.py index 7bb45de5..6b6df922 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -3,6 +3,7 @@ Unit tests for auth classes. Integration tests also exist in tests/client/test_auth.py """ + from urllib.request import parse_keqv_list import pytest diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 5c462915..764f85a2 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -462,8 +462,8 @@ class TestHeaderParamHTML5Formatting: assert expected in request.read() def test_unicode_with_control_character(self): - filename = "hello\x1A\x1B\x1C" - expected = b'filename="hello%1A\x1B%1C"' + filename = "hello\x1a\x1b\x1c" + expected = b'filename="hello%1A\x1b%1C"' files = {"upload": (filename, b"")} request = httpx.Request("GET", "https://www.example.com", files=files) assert expected in request.read() From 7df47ce4d93a06f2c3310cd692b4c2336d7663ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 20:05:55 +0000 Subject: [PATCH 15/21] Bump the python-packages group with 8 updates (#3129) Bumps the python-packages group with 8 updates: | Package | From | To | | --- | --- | --- | | [mkdocs-material](https://github.com/squidfunk/mkdocs-material) | `9.5.6` | `9.5.12` | | [build](https://github.com/pypa/build) | `1.0.3` | `1.1.1` | | [twine](https://github.com/pypa/twine) | `4.0.2` | `5.0.0` | | [coverage[toml]](https://github.com/nedbat/coveragepy) | `7.4.1` | `7.4.3` | | [cryptography](https://github.com/pyca/cryptography) | `42.0.4` | `42.0.5` | | [pytest](https://github.com/pytest-dev/pytest) | `8.0.0` | `8.0.2` | | [ruff](https://github.com/astral-sh/ruff) | `0.1.15` | `0.3.0` | | [uvicorn](https://github.com/encode/uvicorn) | `0.27.0.post1` | `0.27.1` | Updates `mkdocs-material` from 9.5.6 to 9.5.12 - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.6...9.5.12) Updates `build` from 1.0.3 to 1.1.1 - [Release notes](https://github.com/pypa/build/releases) - [Changelog](https://github.com/pypa/build/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/build/compare/1.0.3...1.1.1) Updates `twine` from 4.0.2 to 5.0.0 - [Release notes](https://github.com/pypa/twine/releases) - [Changelog](https://github.com/pypa/twine/blob/main/docs/changelog.rst) - [Commits](https://github.com/pypa/twine/compare/4.0.2...5.0.0) Updates `coverage[toml]` from 7.4.1 to 7.4.3 - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.4.1...7.4.3) Updates `cryptography` from 42.0.4 to 42.0.5 - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.4...42.0.5) Updates `pytest` from 8.0.0 to 8.0.2 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.2) Updates `ruff` from 0.1.15 to 0.3.0 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.1.15...v0.3.0) Updates `uvicorn` from 0.27.0.post1 to 0.27.1 - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.27.0.post1...0.27.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: build dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: twine dependency-type: direct:production update-type: version-update:semver-major dependency-group: python-packages - dependency-name: coverage[toml] dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tom Christie --- requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc72ea12..b9c9588d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,19 +11,19 @@ chardet==5.2.0 # Documentation mkdocs==1.5.3 mkautodoc==0.2.0 -mkdocs-material==9.5.6 +mkdocs-material==9.5.12 # Packaging -build==1.0.3 -twine==4.0.2 +build==1.1.1 +twine==5.0.0 # Tests & Linting -coverage[toml]==7.4.1 -cryptography==42.0.4 +coverage[toml]==7.4.3 +cryptography==42.0.5 mypy==1.8.0 -pytest==8.0.0 -ruff==0.1.15 +pytest==8.0.2 +ruff==0.3.0 trio==0.24.0 trio-typing==0.10.0 trustme==1.1.0 -uvicorn==0.27.0.post1 +uvicorn==0.27.1 From 392dbe45f086d0877bd288c5d68abf860653b680 Mon Sep 17 00:00:00 2001 From: "Michiel W. Beijen" Date: Thu, 21 Mar 2024 11:17:15 +0100 Subject: [PATCH 16/21] Add support for zstd decoding (#3139) This adds support for zstd decoding using the python package zstandard. This is similar to how it is implemented in urllib3. I also chose the optional installation option httpx[zstd] to mimic the same option in urllib3. zstd decoding is similar to brotli, but in benchmarks it is supposed to be even faster. The zstd compression is described in RFC 8878. See https://github.com/encode/httpx/discussions/1986 Co-authored-by: Kamil Monicz --- CHANGELOG.md | 4 +++ README.md | 1 + docs/index.md | 5 ++-- docs/quickstart.md | 6 ++-- httpx/_compat.py | 21 ++++++++++++++ httpx/_decoders.py | 43 +++++++++++++++++++++++++++- httpx/_models.py | 4 +-- pyproject.toml | 3 ++ requirements.txt | 2 +- tests/client/test_client.py | 2 +- tests/client/test_event_hooks.py | 12 ++++---- tests/client/test_headers.py | 16 +++++------ tests/test_asgi.py | 2 +- tests/test_decoders.py | 49 ++++++++++++++++++++++++++++++++ tests/test_main.py | 4 +-- 15 files changed, 148 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85d3bcec..18ded9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +## Added + +* Support for `zstd` content decoding using the python `zstandard` package is added. Installable using `httpx[zstd]`. (#3139) + ### Fixed * Fix `app` type signature in `ASGITransport`. (#3109) diff --git a/README.md b/README.md index 62fb295d..bcba1bb7 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ As well as these optional installs: * `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* * `click` - Command line client support. *(Optional, with `httpx[cli]`)* * `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* +* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* A huge amount of credit is due to `requests` for the API layout that much of this work follows, as well as to `urllib3` for plenty of design diff --git a/docs/index.md b/docs/index.md index 86b6d1cb..387e8504 100644 --- a/docs/index.md +++ b/docs/index.md @@ -119,6 +119,7 @@ As well as these optional installs: * `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* * `click` - Command line client support. *(Optional, with `httpx[cli]`)* * `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* +* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* A huge amount of credit is due to `requests` for the API layout that much of this work follows, as well as to `urllib3` for plenty of design @@ -138,10 +139,10 @@ Or, to include the optional HTTP/2 support, use: $ pip install httpx[http2] ``` -To include the optional brotli decoder support, use: +To include the optional brotli and zstandard decoders support, use: ```shell -$ pip install httpx[brotli] +$ pip install httpx[brotli,zstd] ``` HTTPX requires Python 3.8+ diff --git a/docs/quickstart.md b/docs/quickstart.md index 974119f7..aa203a83 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -100,7 +100,8 @@ b'\n\n\nExample Domain...' Any `gzip` and `deflate` HTTP response encodings will automatically be decoded for you. If `brotlipy` is installed, then the `brotli` response -encoding will also be supported. +encoding will be supported. If `zstandard` is installed, then `zstd` +response encodings will also be supported. For example, to create an image from binary data returned by a request, you can use the following code: @@ -362,7 +363,8 @@ Or stream the text, on a line-by-line basis... HTTPX will use universal line endings, normalising all cases to `\n`. -In some cases you might want to access the raw bytes on the response without applying any HTTP content decoding. In this case any content encoding that the web server has applied such as `gzip`, `deflate`, or `brotli` will not be automatically decoded. +In some cases you might want to access the raw bytes on the response without applying any HTTP content decoding. In this case any content encoding that the web server has applied such as `gzip`, `deflate`, `brotli`, or `zstd` will +not be automatically decoded. ```pycon >>> with httpx.stream("GET", "https://www.example.com") as r: diff --git a/httpx/_compat.py b/httpx/_compat.py index 27ccc682..7d86dced 100644 --- a/httpx/_compat.py +++ b/httpx/_compat.py @@ -3,8 +3,11 @@ The _compat module is used for code which requires branching between different Python environments. It is excluded from the code coverage checks. """ +import re import ssl import sys +from types import ModuleType +from typing import Optional # Brotli support is optional # The C bindings in `brotli` are recommended for CPython. @@ -17,6 +20,24 @@ except ImportError: # pragma: no cover except ImportError: brotli = None +# Zstandard support is optional +zstd: Optional[ModuleType] = None +try: + import zstandard as zstd +except (AttributeError, ImportError, ValueError): # Defensive: + zstd = None +else: + # The package 'zstandard' added the 'eof' property starting + # in v0.18.0 which we require to ensure a complete and + # valid zstd stream was fed into the ZstdDecoder. + # See: https://github.com/urllib3/urllib3/pull/2624 + _zstd_version = tuple( + map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] + ) + if _zstd_version < (0, 18): # Defensive: + zstd = None + + if sys.version_info >= (3, 10) or ssl.OPENSSL_VERSION_INFO >= (1, 1, 0, 7): def set_minimum_tls_version_1_2(context: ssl.SSLContext) -> None: diff --git a/httpx/_decoders.py b/httpx/_decoders.py index f9d3adbb..62f2c0b9 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -11,7 +11,7 @@ import io import typing import zlib -from ._compat import brotli +from ._compat import brotli, zstd from ._exceptions import DecodingError @@ -140,6 +140,44 @@ class BrotliDecoder(ContentDecoder): raise DecodingError(str(exc)) from exc +class ZStandardDecoder(ContentDecoder): + """ + Handle 'zstd' RFC 8878 decoding. + + Requires `pip install zstandard`. + Can be installed as a dependency of httpx using `pip install httpx[zstd]`. + """ + + # inspired by the ZstdDecoder implementation in urllib3 + def __init__(self) -> None: + if zstd is None: # pragma: no cover + raise ImportError( + "Using 'ZStandardDecoder', ..." + "Make sure to install httpx using `pip install httpx[zstd]`." + ) from None + + self.decompressor = zstd.ZstdDecompressor().decompressobj() + + def decode(self, data: bytes) -> bytes: + assert zstd is not None + output = io.BytesIO() + try: + output.write(self.decompressor.decompress(data)) + while self.decompressor.eof and self.decompressor.unused_data: + unused_data = self.decompressor.unused_data + self.decompressor = zstd.ZstdDecompressor().decompressobj() + output.write(self.decompressor.decompress(unused_data)) + except zstd.ZstdError as exc: + raise DecodingError(str(exc)) from exc + return output.getvalue() + + def flush(self) -> bytes: + ret = self.decompressor.flush() # note: this is a no-op + if not self.decompressor.eof: + raise DecodingError("Zstandard data is incomplete") # pragma: no cover + return bytes(ret) + + class MultiDecoder(ContentDecoder): """ Handle the case where multiple encodings have been applied. @@ -323,8 +361,11 @@ SUPPORTED_DECODERS = { "gzip": GZipDecoder, "deflate": DeflateDecoder, "br": BrotliDecoder, + "zstd": ZStandardDecoder, } if brotli is None: SUPPORTED_DECODERS.pop("br") # pragma: no cover +if zstd is None: + SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/httpx/_models.py b/httpx/_models.py index 92b393a2..01d9583b 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -818,7 +818,7 @@ class Response: def iter_bytes(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: """ A byte-iterator over the decoded response content. - This allows us to handle gzip, deflate, and brotli encoded responses. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. """ if hasattr(self, "_content"): chunk_size = len(self._content) if chunk_size is None else chunk_size @@ -918,7 +918,7 @@ class Response: ) -> typing.AsyncIterator[bytes]: """ A byte-iterator over the decoded response content. - This allows us to handle gzip, deflate, and brotli encoded responses. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. """ if hasattr(self, "_content"): chunk_size = len(self._content) if chunk_size is None else chunk_size diff --git a/pyproject.toml b/pyproject.toml index 9e6464c2..c4c18805 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ http2 = [ socks = [ "socksio==1.*", ] +zstd = [ + "zstandard>=0.18.0", +] [project.scripts] httpx = "httpx:main" diff --git a/requirements.txt b/requirements.txt index b9c9588d..3e73fbdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # On the other hand, we're not pinning package dependencies, because our tests # needs to pass with the latest version of the packages. # Reference: https://github.com/encode/httpx/pull/1721#discussion_r661241588 --e .[brotli,cli,http2,socks] +-e .[brotli,cli,http2,socks,zstd] # Optional charset auto-detection # Used in our test cases diff --git a/tests/client/test_client.py b/tests/client/test_client.py index 2951e01b..65783901 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -357,7 +357,7 @@ def test_raw_client_header(): assert response.json() == [ ["Host", "example.org"], ["Accept", "*/*"], - ["Accept-Encoding", "gzip, deflate, br"], + ["Accept-Encoding", "gzip, deflate, br, zstd"], ["Connection", "keep-alive"], ["User-Agent", f"python-httpx/{httpx.__version__}"], ["Example-Header", "example-value"], diff --git a/tests/client/test_event_hooks.py b/tests/client/test_event_hooks.py index 6604dd31..78fb0484 100644 --- a/tests/client/test_event_hooks.py +++ b/tests/client/test_event_hooks.py @@ -36,7 +36,7 @@ def test_event_hooks(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, @@ -87,7 +87,7 @@ async def test_async_event_hooks(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, @@ -144,7 +144,7 @@ def test_event_hooks_with_redirect(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, @@ -159,7 +159,7 @@ def test_event_hooks_with_redirect(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, @@ -201,7 +201,7 @@ async def test_async_event_hooks_with_redirect(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, @@ -216,7 +216,7 @@ async def test_async_event_hooks_with_redirect(): "host": "127.0.0.1:8000", "user-agent": f"python-httpx/{httpx.__version__}", "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=", }, diff --git a/tests/client/test_headers.py b/tests/client/test_headers.py index 264ca0bd..c51e40c3 100755 --- a/tests/client/test_headers.py +++ b/tests/client/test_headers.py @@ -34,7 +34,7 @@ def test_client_header(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "example-header": "example-value", "host": "example.org", @@ -56,7 +56,7 @@ def test_header_merge(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org", "user-agent": "python-myclient/0.2.1", @@ -78,7 +78,7 @@ def test_header_merge_conflicting_headers(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org", "user-agent": f"python-httpx/{httpx.__version__}", @@ -100,7 +100,7 @@ def test_header_update(): assert first_response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org", "user-agent": f"python-httpx/{httpx.__version__}", @@ -111,7 +111,7 @@ def test_header_update(): assert second_response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "another-header": "AThing", "connection": "keep-alive", "host": "example.org", @@ -164,7 +164,7 @@ def test_remove_default_header(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org", } @@ -192,7 +192,7 @@ def test_host_with_auth_and_port_in_url(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org", "user-agent": f"python-httpx/{httpx.__version__}", @@ -215,7 +215,7 @@ def test_host_with_non_default_port_in_url(): assert response.json() == { "headers": { "accept": "*/*", - "accept-encoding": "gzip, deflate, br", + "accept-encoding": "gzip, deflate, br, zstd", "connection": "keep-alive", "host": "example.org:123", "user-agent": f"python-httpx/{httpx.__version__}", diff --git a/tests/test_asgi.py b/tests/test_asgi.py index ccc55266..8b817891 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -157,7 +157,7 @@ async def test_asgi_headers(): "headers": [ ["host", "www.example.org"], ["accept", "*/*"], - ["accept-encoding", "gzip, deflate, br"], + ["accept-encoding", "gzip, deflate, br, zstd"], ["connection", "keep-alive"], ["user-agent", f"python-httpx/{httpx.__version__}"], ] diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 73644e04..bcbb18bb 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -1,10 +1,12 @@ from __future__ import annotations +import io import typing import zlib import chardet import pytest +import zstandard as zstd import httpx @@ -73,6 +75,53 @@ def test_brotli(): assert response.content == body +def test_zstd(): + body = b"test 123" + compressed_body = zstd.compress(body) + + headers = [(b"Content-Encoding", b"zstd")] + response = httpx.Response( + 200, + headers=headers, + content=compressed_body, + ) + assert response.content == body + + +def test_zstd_decoding_error(): + compressed_body = "this_is_not_zstd_compressed_data" + + headers = [(b"Content-Encoding", b"zstd")] + with pytest.raises(httpx.DecodingError): + httpx.Response( + 200, + headers=headers, + content=compressed_body, + ) + + +def test_zstd_multiframe(): + # test inspired by urllib3 test suite + data = ( + # Zstandard frame + zstd.compress(b"foo") + # skippable frame (must be ignored) + + bytes.fromhex( + "50 2A 4D 18" # Magic_Number (little-endian) + "07 00 00 00" # Frame_Size (little-endian) + "00 00 00 00 00 00 00" # User_Data + ) + # Zstandard frame + + zstd.compress(b"bar") + ) + compressed_body = io.BytesIO(data) + + headers = [(b"Content-Encoding", b"zstd")] + response = httpx.Response(200, headers=headers, content=compressed_body) + response.read() + assert response.content == b"foobar" + + def test_multi(): body = b"test 123" diff --git a/tests/test_main.py b/tests/test_main.py index 67eeb0d2..feb796e1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -129,7 +129,7 @@ def test_verbose(server): "GET / HTTP/1.1", f"Host: {server.url.netloc.decode('ascii')}", "Accept: */*", - "Accept-Encoding: gzip, deflate, br", + "Accept-Encoding: gzip, deflate, br, zstd", "Connection: keep-alive", f"User-Agent: python-httpx/{httpx.__version__}", "", @@ -154,7 +154,7 @@ def test_auth(server): "GET / HTTP/1.1", f"Host: {server.url.netloc.decode('ascii')}", "Accept: */*", - "Accept-Encoding: gzip, deflate, br", + "Accept-Encoding: gzip, deflate, br, zstd", "Connection: keep-alive", f"User-Agent: python-httpx/{httpx.__version__}", "Authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=", From 45bb65bba12360d94b8c512e6b13ac4b775a402d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 6 Apr 2024 08:30:16 +0200 Subject: [PATCH 17/21] Document 'target' extension (#3160) --- docs/advanced/extensions.md | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/advanced/extensions.md b/docs/advanced/extensions.md index fa317eeb..9eafebd4 100644 --- a/docs/advanced/extensions.md +++ b/docs/advanced/extensions.md @@ -138,6 +138,47 @@ response = client.get( This extension is how the `httpx` timeouts are implemented, ensuring that the timeout values are associated with the request instance and passed throughout the stack. You shouldn't typically be working with this extension directly, but use the higher level `timeout` API instead. +### `"target"` + +The target that is used as [the HTTP target instead of the URL path](https://datatracker.ietf.org/doc/html/rfc2616#section-5.1.2). + +This enables support constructing requests that would otherwise be unsupported. + +* URL paths with non-standard escaping applied. +* Forward proxy requests using an absolute URI. +* Tunneling proxy requests using `CONNECT` with hostname as the target. +* Server-wide `OPTIONS *` requests. + +Some examples: + +Using the 'target' extension to send requests without the standard path escaping rules... + +```python +# Typically a request to "https://www.example.com/test^path" would +# connect to "www.example.com" and send an HTTP/1.1 request like... +# +# GET /test%5Epath HTTP/1.1 +# +# Using the target extension we can include the literal '^'... +# +# GET /test^path HTTP/1.1 +# +# Note that requests must still be valid HTTP requests. +# For example including whitespace in the target will raise a `LocalProtocolError`. +extensions = {"target": b"/test^path"} +response = httpx.get("https://www.example.com", extensions=extensions) +``` + +The `target` extension also allows server-wide `OPTIONS *` requests to be constructed... + +```python +# This will send the following request... +# +# CONNECT * HTTP/1.1 +extensions = {"target": b"*"} +response = httpx.request("CONNECT", "https://www.example.com", extensions=extensions) +``` + ## Response Extensions ### `"http_version"` From 5bb2ea0f4e1274730ef85f50f5a6bf07eb74eef4 Mon Sep 17 00:00:00 2001 From: Hugo Cachitas Date: Sat, 6 Apr 2024 12:55:26 +0100 Subject: [PATCH 18/21] Update URL.__init__ signature (#3159) --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 3f9878c7..d01cc649 100644 --- a/docs/api.md +++ b/docs/api.md @@ -114,7 +114,7 @@ what gets sent over the wire.* 'example.org' ``` -* `def __init__(url, allow_relative=False, params=None)` +* `def __init__(url, **kwargs)` * `.scheme` - **str** * `.authority` - **str** * `.host` - **str** From 7354ed70ceb1a0f072af82e2cb784ef6b2512ed3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 21:38:43 +0100 Subject: [PATCH 19/21] Bump the python-packages group with 8 updates (#3156) --- requirements.txt | 16 ++++++++-------- tests/conftest.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e73fbdb..a119fb98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,19 +11,19 @@ chardet==5.2.0 # Documentation mkdocs==1.5.3 mkautodoc==0.2.0 -mkdocs-material==9.5.12 +mkdocs-material==9.5.16 # Packaging -build==1.1.1 +build==1.2.1 twine==5.0.0 # Tests & Linting -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 cryptography==42.0.5 -mypy==1.8.0 -pytest==8.0.2 -ruff==0.3.0 -trio==0.24.0 +mypy==1.9.0 +pytest==8.1.1 +ruff==0.3.4 +trio==0.25.0 trio-typing==0.10.0 trustme==1.1.0 -uvicorn==0.27.1 +uvicorn==0.29.0 diff --git a/tests/conftest.py b/tests/conftest.py index 1bcb6a42..5c4a6ae5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -236,7 +236,7 @@ class TestServer(Server): def install_signal_handlers(self) -> None: # Disable the default installation of handlers for signals such as SIGTERM, # because it can only be done in the main thread. - pass + pass # pragma: nocover async def serve(self, sockets=None): self.restart_requested = asyncio.Event() From 4b85e6c3898b94e686b427afd83138c87520b479 Mon Sep 17 00:00:00 2001 From: "Michiel W. Beijen" Date: Fri, 12 Apr 2024 08:11:12 +0200 Subject: [PATCH 20/21] Docs: fix small typos in Extensions doc (#3138) Co-authored-by: Tom Christie --- docs/advanced/extensions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/extensions.md b/docs/advanced/extensions.md index 9eafebd4..d9208ccd 100644 --- a/docs/advanced/extensions.md +++ b/docs/advanced/extensions.md @@ -2,7 +2,7 @@ Request and response extensions provide a untyped space where additional information may be added. -Extensions should be used for features that may not be available on all transports, and that do not fit neatly into [the simplified request/response model](https://www.encode.io/httpcore/extensions/) that the underlying `httpcore` pacakge uses as it's API. +Extensions should be used for features that may not be available on all transports, and that do not fit neatly into [the simplified request/response model](https://www.encode.io/httpcore/extensions/) that the underlying `httpcore` package uses as its API. Several extensions are supported on the request: @@ -239,4 +239,4 @@ with httpx.stream("GET", "https://www.example.com") as response: ssl_object = network_stream.get_extra_info("ssl_object") print("TLS version", ssl_object.version()) -``` \ No newline at end of file +``` From 2f5ae50726b1a2ca78340a25054f81cf4ed926c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 17:56:17 +0100 Subject: [PATCH 21/21] Bump the python-packages group with 6 updates (#3185) --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index a119fb98..99d13174 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,20 +9,20 @@ chardet==5.2.0 # Documentation -mkdocs==1.5.3 +mkdocs==1.6.0 mkautodoc==0.2.0 -mkdocs-material==9.5.16 +mkdocs-material==9.5.20 # Packaging build==1.2.1 twine==5.0.0 # Tests & Linting -coverage[toml]==7.4.4 +coverage[toml]==7.5.0 cryptography==42.0.5 -mypy==1.9.0 -pytest==8.1.1 -ruff==0.3.4 +mypy==1.10.0 +pytest==8.2.0 +ruff==0.4.2 trio==0.25.0 trio-typing==0.10.0 trustme==1.1.0