From 45bb65bba12360d94b8c512e6b13ac4b775a402d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 6 Apr 2024 08:30:16 +0200 Subject: [PATCH 1/9] Document 'target' extension (#3160) --- docs/advanced/extensions.md | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/advanced/extensions.md b/docs/advanced/extensions.md index fa317eeb..9eafebd4 100644 --- a/docs/advanced/extensions.md +++ b/docs/advanced/extensions.md @@ -138,6 +138,47 @@ response = client.get( This extension is how the `httpx` timeouts are implemented, ensuring that the timeout values are associated with the request instance and passed throughout the stack. You shouldn't typically be working with this extension directly, but use the higher level `timeout` API instead. +### `"target"` + +The target that is used as [the HTTP target instead of the URL path](https://datatracker.ietf.org/doc/html/rfc2616#section-5.1.2). + +This enables support constructing requests that would otherwise be unsupported. + +* URL paths with non-standard escaping applied. +* Forward proxy requests using an absolute URI. +* Tunneling proxy requests using `CONNECT` with hostname as the target. +* Server-wide `OPTIONS *` requests. + +Some examples: + +Using the 'target' extension to send requests without the standard path escaping rules... + +```python +# Typically a request to "https://www.example.com/test^path" would +# connect to "www.example.com" and send an HTTP/1.1 request like... +# +# GET /test%5Epath HTTP/1.1 +# +# Using the target extension we can include the literal '^'... +# +# GET /test^path HTTP/1.1 +# +# Note that requests must still be valid HTTP requests. +# For example including whitespace in the target will raise a `LocalProtocolError`. +extensions = {"target": b"/test^path"} +response = httpx.get("https://www.example.com", extensions=extensions) +``` + +The `target` extension also allows server-wide `OPTIONS *` requests to be constructed... + +```python +# This will send the following request... +# +# CONNECT * HTTP/1.1 +extensions = {"target": b"*"} +response = httpx.request("CONNECT", "https://www.example.com", extensions=extensions) +``` + ## Response Extensions ### `"http_version"` From 5bb2ea0f4e1274730ef85f50f5a6bf07eb74eef4 Mon Sep 17 00:00:00 2001 From: Hugo Cachitas Date: Sat, 6 Apr 2024 12:55:26 +0100 Subject: [PATCH 2/9] Update URL.__init__ signature (#3159) --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 3f9878c7..d01cc649 100644 --- a/docs/api.md +++ b/docs/api.md @@ -114,7 +114,7 @@ what gets sent over the wire.* 'example.org' ``` -* `def __init__(url, allow_relative=False, params=None)` +* `def __init__(url, **kwargs)` * `.scheme` - **str** * `.authority` - **str** * `.host` - **str** From 7354ed70ceb1a0f072af82e2cb784ef6b2512ed3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 21:38:43 +0100 Subject: [PATCH 3/9] Bump the python-packages group with 8 updates (#3156) --- requirements.txt | 16 ++++++++-------- tests/conftest.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e73fbdb..a119fb98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,19 +11,19 @@ chardet==5.2.0 # Documentation mkdocs==1.5.3 mkautodoc==0.2.0 -mkdocs-material==9.5.12 +mkdocs-material==9.5.16 # Packaging -build==1.1.1 +build==1.2.1 twine==5.0.0 # Tests & Linting -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 cryptography==42.0.5 -mypy==1.8.0 -pytest==8.0.2 -ruff==0.3.0 -trio==0.24.0 +mypy==1.9.0 +pytest==8.1.1 +ruff==0.3.4 +trio==0.25.0 trio-typing==0.10.0 trustme==1.1.0 -uvicorn==0.27.1 +uvicorn==0.29.0 diff --git a/tests/conftest.py b/tests/conftest.py index 1bcb6a42..5c4a6ae5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -236,7 +236,7 @@ class TestServer(Server): def install_signal_handlers(self) -> None: # Disable the default installation of handlers for signals such as SIGTERM, # because it can only be done in the main thread. - pass + pass # pragma: nocover async def serve(self, sockets=None): self.restart_requested = asyncio.Event() From 4b85e6c3898b94e686b427afd83138c87520b479 Mon Sep 17 00:00:00 2001 From: "Michiel W. Beijen" Date: Fri, 12 Apr 2024 08:11:12 +0200 Subject: [PATCH 4/9] Docs: fix small typos in Extensions doc (#3138) Co-authored-by: Tom Christie --- docs/advanced/extensions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/extensions.md b/docs/advanced/extensions.md index 9eafebd4..d9208ccd 100644 --- a/docs/advanced/extensions.md +++ b/docs/advanced/extensions.md @@ -2,7 +2,7 @@ Request and response extensions provide a untyped space where additional information may be added. -Extensions should be used for features that may not be available on all transports, and that do not fit neatly into [the simplified request/response model](https://www.encode.io/httpcore/extensions/) that the underlying `httpcore` pacakge uses as it's API. +Extensions should be used for features that may not be available on all transports, and that do not fit neatly into [the simplified request/response model](https://www.encode.io/httpcore/extensions/) that the underlying `httpcore` package uses as its API. Several extensions are supported on the request: @@ -239,4 +239,4 @@ with httpx.stream("GET", "https://www.example.com") as response: ssl_object = network_stream.get_extra_info("ssl_object") print("TLS version", ssl_object.version()) -``` \ No newline at end of file +``` From 2f5ae50726b1a2ca78340a25054f81cf4ed926c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 17:56:17 +0100 Subject: [PATCH 5/9] Bump the python-packages group with 6 updates (#3185) --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index a119fb98..99d13174 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,20 +9,20 @@ chardet==5.2.0 # Documentation -mkdocs==1.5.3 +mkdocs==1.6.0 mkautodoc==0.2.0 -mkdocs-material==9.5.16 +mkdocs-material==9.5.20 # Packaging build==1.2.1 twine==5.0.0 # Tests & Linting -coverage[toml]==7.4.4 +coverage[toml]==7.5.0 cryptography==42.0.5 -mypy==1.9.0 -pytest==8.1.1 -ruff==0.3.4 +mypy==1.10.0 +pytest==8.2.0 +ruff==0.4.2 trio==0.25.0 trio-typing==0.10.0 trustme==1.1.0 From be56b747350008b7b0456242691919f9086c0f32 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 2 May 2024 18:07:09 +0800 Subject: [PATCH 6/9] Fix doc links for making requests directly to WSGI/ASGI apps (#3186) --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bcba1bb7..d5d21487 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ HTTPX builds on the well-established usability of `requests`, and gives you: * An integrated command-line client. * HTTP/1.1 [and HTTP/2 support](https://www.python-httpx.org/http2/). * Standard synchronous interface, but with [async support if you need it](https://www.python-httpx.org/async/). -* Ability to make requests directly to [WSGI applications](https://www.python-httpx.org/advanced/#calling-into-python-web-apps) or [ASGI applications](https://www.python-httpx.org/async/#calling-into-python-web-apps). +* Ability to make requests directly to [WSGI applications](https://www.python-httpx.org/advanced/transports/#wsgi-transport) or [ASGI applications](https://www.python-httpx.org/advanced/transports/#asgi-transport). * Strict timeouts everywhere. * Fully type annotated. * 100% test coverage. diff --git a/docs/index.md b/docs/index.md index 387e8504..c2210bc7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -68,7 +68,7 @@ HTTPX builds on the well-established usability of `requests`, and gives you: * A broadly [requests-compatible API](compatibility.md). * Standard synchronous interface, but with [async support if you need it](async.md). * HTTP/1.1 [and HTTP/2 support](http2.md). -* Ability to make requests directly to [WSGI applications](async.md#calling-into-python-web-apps) or [ASGI applications](async.md#calling-into-python-web-apps). +* Ability to make requests directly to [WSGI applications](advanced/transports.md#wsgi-transport) or [ASGI applications](advanced/transports.md#asgi-transport). * Strict timeouts everywhere. * Fully type annotated. * 100% test coverage. From a7092af2fda78d92daaad7627e4cf0cf5e94b019 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 3 May 2024 01:09:08 +0100 Subject: [PATCH 7/9] Resolve queryparam quoting (#3187) --- httpx/_urlparse.py | 36 ++++++---------------------- tests/models/test_url.py | 51 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/httpx/_urlparse.py b/httpx/_urlparse.py index 232269ee..883f0895 100644 --- a/httpx/_urlparse.py +++ b/httpx/_urlparse.py @@ -406,44 +406,22 @@ def normalize_path(path: str) -> str: return "/".join(output) -def percent_encode(char: str) -> str: - """ - Replace a single character with the percent-encoded representation. - - Characters outside the ASCII range are represented with their a percent-encoded - representation of their UTF-8 byte sequence. - - For example: - - percent_encode(" ") == "%20" - """ - return "".join([f"%{byte:02x}" for byte in char.encode("utf-8")]).upper() - - -def is_safe(string: str, safe: str = "/") -> bool: - """ - Determine if a given string is already quote-safe. - """ - NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe + "%" - - # All characters must already be non-escaping or '%' - for char in string: - if char not in NON_ESCAPED_CHARS: - return False - - return True +def PERCENT(string: str) -> str: + return "".join([f"%{byte:02X}" for byte in string.encode("utf-8")]) def percent_encoded(string: str, safe: str = "/") -> str: """ Use percent-encoding to quote a string. """ - if is_safe(string, safe=safe): + NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe + + # Fast path for strings that don't need escaping. + if not string.rstrip(NON_ESCAPED_CHARS): return string - NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe return "".join( - [char if char in NON_ESCAPED_CHARS else percent_encode(char) for char in string] + [char if char in NON_ESCAPED_CHARS else PERCENT(char) for char in string] ) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index 79e1605a..32571238 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -229,6 +229,11 @@ def test_url_normalized_host(): assert url.host == "example.com" +def test_url_percent_escape_host(): + url = httpx.URL("https://exam%le.com/") + assert url.host == "exam%25le.com" + + def test_url_ipv4_like_host(): """rare host names used to quality as IPv4""" url = httpx.URL("https://023b76x43144/") @@ -278,24 +283,64 @@ def test_url_leading_dot_prefix_on_relative_url(): assert url.path == "../abc" -# Tests for optional percent encoding +# Tests for query parameter percent encoding. +# +# Percent-encoding in `params={}` should match browser form behavior. -def test_param_requires_encoding(): +def test_param_with_space(): + # Params passed as form key-value pairs should be escaped. url = httpx.URL("http://webservice", params={"u": "with spaces"}) assert str(url) == "http://webservice?u=with%20spaces" def test_param_does_not_require_encoding(): + # Params passed as form key-value pairs should be escaped. + url = httpx.URL("http://webservice", params={"u": "%"}) + assert str(url) == "http://webservice?u=%25" + + +def test_param_with_percent_encoded(): + # Params passed as form key-value pairs should always be escaped, + # even if they include a valid escape sequence. + # We want to match browser form behaviour here. url = httpx.URL("http://webservice", params={"u": "with%20spaces"}) - assert str(url) == "http://webservice?u=with%20spaces" + assert str(url) == "http://webservice?u=with%2520spaces" def test_param_with_existing_escape_requires_encoding(): + # Params passed as form key-value pairs should always be escaped, + # even if they include a valid escape sequence. + # We want to match browser form behaviour here. url = httpx.URL("http://webservice", params={"u": "http://example.com?q=foo%2Fa"}) assert str(url) == "http://webservice?u=http%3A%2F%2Fexample.com%3Fq%3Dfoo%252Fa" +# Tests for query parameter percent encoding. +# +# Percent-encoding in `url={}` should match browser URL bar behavior. + + +def test_query_with_existing_percent_encoding(): + # Valid percent encoded sequences should not be double encoded. + url = httpx.URL("http://webservice?u=phrase%20with%20spaces") + assert str(url) == "http://webservice?u=phrase%20with%20spaces" + + +def test_query_requiring_percent_encoding(): + # Characters that require percent encoding should be encoded. + url = httpx.URL("http://webservice?u=phrase with spaces") + assert str(url) == "http://webservice?u=phrase%20with%20spaces" + + +def test_query_with_mixed_percent_encoding(): + # When a mix of encoded and unencoded characters are present, + # characters that require percent encoding should be encoded, + # while existing sequences should not be double encoded. + url = httpx.URL("http://webservice?u=phrase%20with spaces") + assert str(url) == "http://webservice?u=phrase%20with%20spaces" + + # Tests for invalid URLs From fa6dac8383541669f42086d4f07c71b0f8d6a279 Mon Sep 17 00:00:00 2001 From: Shiny Date: Mon, 6 May 2024 00:24:16 +0800 Subject: [PATCH 8/9] Removed leading $ from cli code blocks (#3174) Co-authored-by: Tom Christie --- README.md | 8 ++++---- docs/advanced/proxies.md | 2 +- docs/contributing.md | 18 +++++++++--------- docs/http2.md | 2 +- docs/index.md | 10 +++++----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d5d21487..5e459a28 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ and async APIs**. Install HTTPX using pip: ```shell -$ pip install httpx +pip install httpx ``` Now, let's get started: @@ -43,7 +43,7 @@ Now, let's get started: Or, using the command-line client. ```shell -$ pip install 'httpx[cli]' # The command line client is an optional dependency. +pip install 'httpx[cli]' # The command line client is an optional dependency. ``` Which now allows us to use HTTPX directly from the command-line... @@ -94,13 +94,13 @@ Plus all the standard features of `requests`... Install with pip: ```shell -$ pip install httpx +pip install httpx ``` Or, to include the optional HTTP/2 support, use: ```shell -$ pip install httpx[http2] +pip install httpx[http2] ``` HTTPX requires Python 3.8+. diff --git a/docs/advanced/proxies.md b/docs/advanced/proxies.md index 2a6b7d5f..f1ee3ec8 100644 --- a/docs/advanced/proxies.md +++ b/docs/advanced/proxies.md @@ -73,7 +73,7 @@ This is an optional feature that requires an additional third-party library be i You can install SOCKS support using `pip`: ```shell -$ pip install httpx[socks] +pip install httpx[socks] ``` You can now configure a client to make requests via a proxy using the SOCKS protocol: diff --git a/docs/contributing.md b/docs/contributing.md index 0d3ad5f1..110a127c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -46,14 +46,14 @@ Then clone your fork with the following command replacing `YOUR-USERNAME` with your GitHub username: ```shell -$ git clone https://github.com/YOUR-USERNAME/httpx +git clone https://github.com/YOUR-USERNAME/httpx ``` You can now install the project and its dependencies using: ```shell -$ cd httpx -$ scripts/install +cd httpx +scripts/install ``` ## Testing and Linting @@ -64,7 +64,7 @@ and documentation building workflow. To run the tests, use: ```shell -$ scripts/test +scripts/test ``` !!! warning @@ -76,19 +76,19 @@ Any additional arguments will be passed to `pytest`. See the [pytest documentati For example, to run a single test script: ```shell -$ scripts/test tests/test_multipart.py +scripts/test tests/test_multipart.py ``` To run the code auto-formatting: ```shell -$ scripts/lint +scripts/lint ``` Lastly, to run code checks separately (they are also run as part of `scripts/test`), run: ```shell -$ scripts/check +scripts/check ``` ## Documenting @@ -98,7 +98,7 @@ Documentation pages are located under the `docs/` folder. To run the documentation site locally (useful for previewing changes), use: ```shell -$ scripts/docs +scripts/docs ``` ## Resolving Build / CI Failures @@ -122,7 +122,7 @@ This job failing means there is either a code formatting issue or type-annotatio You can look at the job output to figure out why it's failed or within a shell run: ```shell -$ scripts/check +scripts/check ``` It may be worth it to run `$ scripts/lint` to attempt auto-formatting the code diff --git a/docs/http2.md b/docs/http2.md index 3cab09d9..434606c4 100644 --- a/docs/http2.md +++ b/docs/http2.md @@ -28,7 +28,7 @@ trying out our HTTP/2 support. You can do so by first making sure to install the optional HTTP/2 dependencies... ```shell -$ pip install httpx[http2] +pip install httpx[http2] ``` And then instantiating a client with HTTP/2 support enabled: diff --git a/docs/index.md b/docs/index.md index c2210bc7..98bf0fd6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,7 +28,7 @@ HTTPX is a fully featured HTTP client for Python 3, which provides sync and asyn Install HTTPX using pip: ```shell -$ pip install httpx +pip install httpx ``` Now, let's get started: @@ -50,7 +50,7 @@ Or, using the command-line client. ```shell # The command line client is an optional dependency. -$ pip install 'httpx[cli]' +pip install 'httpx[cli]' ``` Which now allows us to use HTTPX directly from the command-line... @@ -130,19 +130,19 @@ inspiration around the lower-level networking details. Install with pip: ```shell -$ pip install httpx +pip install httpx ``` Or, to include the optional HTTP/2 support, use: ```shell -$ pip install httpx[http2] +pip install httpx[http2] ``` To include the optional brotli and zstandard decoders support, use: ```shell -$ pip install httpx[brotli,zstd] +pip install httpx[brotli,zstd] ``` HTTPX requires Python 3.8+ From 88a81c5d31a4a8b4bd0a860dedd3bb12dc09ff86 Mon Sep 17 00:00:00 2001 From: manav-a <33183958+manav-a@users.noreply.github.com> Date: Fri, 10 May 2024 03:42:50 -0700 Subject: [PATCH 9/9] [fix] Use proxy ssl context consistently (#3175) Co-authored-by: Kar Petrosyan <92274156+karpetrosyan@users.noreply.github.com> --- httpx/_transports/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index bcc8bf42..33db416d 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -303,6 +303,7 @@ class AsyncHTTPTransport(AsyncBaseTransport): ), proxy_auth=proxy.raw_auth, proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, ssl_context=ssl_context, max_connections=limits.max_connections, max_keepalive_connections=limits.max_keepalive_connections,