encode/httpx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transport API (#963)

Browse Source 
* Deprecate Client arg 'dispatch' and use 'transport'

* Remove line in test from coverage

* Document custom transports

* _dispatch > _transports

Also rename *Dispatch classes to *Transport and added aliases

* Fix linting issues

* Missed one _transports import

* Promote URLLib3Transport to public API

* Remove duplicate arg doc

* Assert that urllib3 is imported to use URLLib3Transport

* `AsyncClient`, not asynchronous `Client`

* Add warning category to warn calls

* Update docs/advanced.md

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>

* Add warn_deprecated utility function

* Amend docs references to dispatch

* Add concrete implementation example

* Clearer transport implementation description

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>
This commit is contained in:
Yeray Diaz Diaz 2020-05-21 12:22:17 +01:00 committed by GitHub
parent ba073c8a46
commit d2816c9c48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 215 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
docs/advanced.md
View File

@ -173,7 +173,7 @@ with httpx.Client(app=app, base_url="http://testserver") as client:
assert r.text == "Hello World!"
```
For some more complex cases you might need to customize the WSGI dispatch. This allows you to:
For some more complex cases you might need to customize the WSGI transport. This allows you to:
* Inspect 500 error responses rather than raise exceptions by setting `raise_app_exceptions=False`.
* Mount the WSGI application at a subpath by setting `script_name` (WSGI).
@ -183,8 +183,8 @@ For example:
```python
# Instantiate a client that makes WSGI requests with a client IP of "1.2.3.4".
dispatch = httpx.WSGIDispatch(app=app, remote_addr="1.2.3.4")
with httpx.Client(dispatch=dispatch, base_url="http://testserver") as client:
transport = httpx.WSGITransport(app=app, remote_addr="1.2.3.4")
with httpx.Client(transport=transport, base_url="http://testserver") as client:
...
```
@ -619,3 +619,56 @@ If you do need to make HTTPS connections to a local server, for example to test
>>> r
Response <200 OK>
```
## Custom Transports
HTTPX's `Client` also accepts a `transport` argument. This argument allows you
to provide a custom Transport object that will be used to perform the actual
sending of the requests.
A transport instance must implement the Transport API defined by
[`httpcore`](https://www.encode.io/httpcore/api/). You
should either subclass `httpcore.AsyncHTTPTransport` to implement a transport to
use with `AsyncClient`, or subclass `httpcore.SyncHTTPTransport` to implement a
transport to use with `Client`.
For example, HTTPX ships with a transport that uses the excellent
[`urllib3` library](https://urllib3.readthedocs.io/en/latest/):
```python
>>> import httpx
>>> client = httpx.Client(transport=httpx.URLLib3Transport())
>>> client.get("https://example.org")
<Response [200 OK]>
```
A complete example of a transport implementation would be:
```python
import json
import httpcore
import httpx
class JSONEchoTransport(httpcore.SyncHTTPTransport):
"""
A mock transport that returns a JSON response containing the request body.
"""
def request(self, method, url, headers=None, stream=None, timeout=None):
body = b"".join(stream).decode("utf-8")
content = json.dumps({"body": body}).encode("utf-8")
stream = httpcore.SyncByteStream([content])
headers = [(b"content-type", b"application/json")]
return b"HTTP/1.1", 200, b"OK", headers, stream
```
Which we can use in the same way:
```python
>>> client = httpx.Client(transport=JSONEchoTransport())
>>> response = client.post("https://httpbin.org/post", data="Hello, world!")
>>> response.json()
{'body': 'Hello, world!'}
```

6
docs/async.md
View File

@ -165,7 +165,7 @@ We can make requests directly against the application, like so:
... assert r.text == "Hello World!"
```
For some more complex cases you might need to customise the ASGI dispatch. This allows you to:
For some more complex cases you might need to customise the ASGI transport. This allows you to:
* Inspect 500 error responses rather than raise exceptions by setting `raise_app_exceptions=False`.
* Mount the ASGI application at a subpath by setting `root_path`.
@ -176,8 +176,8 @@ For example:
```python
# Instantiate a client that makes ASGI requests with a client IP of "1.2.3.4",
# on port 123.
dispatch = httpx.ASGIDispatch(app=app, client=("1.2.3.4", 123))
async with httpx.AsyncClient(dispatch=dispatch, base_url="http://testserver") as client:
transport = httpx.ASGITransport(app=app, client=("1.2.3.4", 123))
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
...
```

8
httpx/__init__.py
View File

@ -3,8 +3,6 @@ from ._api import delete, get, head, options, patch, post, put, request, stream
from ._auth import Auth, BasicAuth, DigestAuth
from ._client import AsyncClient, Client
from ._config import PoolLimits, Proxy, Timeout
from ._dispatch.asgi import ASGIDispatch
from ._dispatch.wsgi import WSGIDispatch
from ._exceptions import (
ConnectTimeout,
CookieConflict,
@ -27,6 +25,9 @@ from ._exceptions import (
)
from ._models import URL, Cookies, Headers, QueryParams, Request, Response
from ._status_codes import StatusCode, codes
from ._transports.asgi import ASGIDispatch, ASGITransport
from ._transports.urllib3 import URLLib3Transport
from ._transports.wsgi import WSGIDispatch, WSGITransport
__all__ = [
"__description__",
@ -44,6 +45,7 @@ __all__ = [
"stream",
"codes",
"ASGIDispatch",
"ASGITransport",
"AsyncClient",
"Auth",
"BasicAuth",
@ -71,6 +73,7 @@ __all__ = [
"TooManyRedirects",
"WriteTimeout",
"URL",
"URLLib3Transport",
"StatusCode",
"Cookies",
"Headers",
@ -79,4 +82,5 @@ __all__ = [
"Response",
"DigestAuth",
"WSGIDispatch",
"WSGITransport",
]

97
httpx/_client.py
View File

@ -18,11 +18,11 @@ from ._config import (
UnsetType,
)
from ._content_streams import ContentStream
from ._dispatch.asgi import ASGIDispatch
from ._dispatch.wsgi import WSGIDispatch
from ._exceptions import HTTPError, InvalidURL, RequestBodyUnavailable, TooManyRedirects
from ._models import URL, Cookies, Headers, Origin, QueryParams, Request, Response
from ._status_codes import codes
from ._transports.asgi import ASGITransport
from ._transports.wsgi import WSGITransport
from ._types import (
AuthTypes,
CertTypes,
@ -41,6 +41,7 @@ from ._utils import (
get_environment_proxies,
get_logger,
should_not_be_proxied,
warn_deprecated,
)
logger = get_logger(__name__)
@ -91,7 +92,7 @@ class BaseClient:
return {"all": proxy}
elif isinstance(proxies, httpcore.AsyncHTTPTransport): # pragma: nocover
raise RuntimeError(
"Passing a dispatcher instance to 'proxies=' is no longer "
"Passing a transport instance to 'proxies=' is no longer "
"supported. Use `httpx.Proxy() instead.`"
)
else:
@ -102,7 +103,7 @@ class BaseClient:
new_proxies[str(key)] = proxy
elif isinstance(value, httpcore.AsyncHTTPTransport): # pragma: nocover
raise RuntimeError(
"Passing a dispatcher instance to 'proxies=' is "
"Passing a transport instance to 'proxies=' is "
"no longer supported. Use `httpx.Proxy() instead.`"
)
return new_proxies
@ -417,8 +418,9 @@ class Client(BaseClient):
that should be followed.
* **base_url** - *(optional)* A URL to use as the base when building
request URLs.
* **dispatch** - *(optional)* A dispatch class to use for sending requests
* **transport** - *(optional)* A transport class to use for sending requests
over the network.
* **dispatch** - *(optional)* A deprecated alias for transport.
* **app** - *(optional)* An WSGI application to send requests to,
rather than sending actual network requests.
* **trust_env** - *(optional)* Enables or disables usage of environment
@ -439,6 +441,7 @@ class Client(BaseClient):
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
max_redirects: int = DEFAULT_MAX_REDIRECTS,
base_url: URLTypes = None,
transport: httpcore.SyncHTTPTransport = None,
dispatch: httpcore.SyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
@ -456,16 +459,24 @@ class Client(BaseClient):
proxy_map = self.get_proxy_map(proxies, trust_env)
self.dispatch = self.init_dispatch(
if dispatch is not None:
warn_deprecated(
"The dispatch argument is deprecated since v0.13 and will be "
"removed in a future release, please use 'transport'"
)
if transport is None:
transport = dispatch
self.transport = self.init_transport(
verify=verify,
cert=cert,
pool_limits=pool_limits,
dispatch=dispatch,
transport=transport,
app=app,
trust_env=trust_env,
)
self.proxies: typing.Dict[str, httpcore.SyncHTTPTransport] = {
key: self.init_proxy_dispatch(
key: self.init_proxy_transport(
proxy,
verify=verify,
cert=cert,
@ -475,20 +486,20 @@ class Client(BaseClient):
for key, proxy in proxy_map.items()
}
def init_dispatch(
def init_transport(
self,
verify: VerifyTypes = True,
cert: CertTypes = None,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
dispatch: httpcore.SyncHTTPTransport = None,
transport: httpcore.SyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
) -> httpcore.SyncHTTPTransport:
if dispatch is not None:
return dispatch
if transport is not None:
return transport
if app is not None:
return WSGIDispatch(app=app)
return WSGITransport(app=app)
ssl_context = SSLConfig(
verify=verify, cert=cert, trust_env=trust_env
@ -502,7 +513,7 @@ class Client(BaseClient):
max_connections=max_connections,
)
def init_proxy_dispatch(
def init_proxy_transport(
self,
proxy: Proxy,
verify: VerifyTypes = True,
@ -525,7 +536,7 @@ class Client(BaseClient):
max_connections=max_connections,
)
def dispatcher_for_url(self, url: URL) -> httpcore.SyncHTTPTransport:
def transport_for_url(self, url: URL) -> httpcore.SyncHTTPTransport:
"""
Returns the transport instance that should be used for a given URL.
This will either be the standard connection pool, or a proxy.
@ -545,10 +556,10 @@ class Client(BaseClient):
)
for proxy_key in proxy_keys:
if proxy_key and proxy_key in self.proxies:
dispatcher = self.proxies[proxy_key]
return dispatcher
transport = self.proxies[proxy_key]
return transport
return self.dispatch
return self.transport
def request(
self,
@ -680,7 +691,7 @@ class Client(BaseClient):
Sends a single request, without handling any redirections.
"""
dispatcher = self.dispatcher_for_url(request.url)
transport = self.transport_for_url(request.url)
try:
(
@ -689,7 +700,7 @@ class Client(BaseClient):
reason_phrase,
headers,
stream,
) = dispatcher.request(
) = transport.request(
request.method.encode(),
request.url.raw,
headers=request.headers.raw,
@ -892,7 +903,7 @@ class Client(BaseClient):
)
def close(self) -> None:
self.dispatch.close()
self.transport.close()
for proxy in self.proxies.values():
proxy.close()
@ -949,8 +960,9 @@ class AsyncClient(BaseClient):
that should be followed.
* **base_url** - *(optional)* A URL to use as the base when building
request URLs.
* **dispatch** - *(optional)* A dispatch class to use for sending requests
* **transport** - *(optional)* A transport class to use for sending requests
over the network.
* **dispatch** - *(optional)* A deprecated alias for transport.
* **app** - *(optional)* An ASGI application to send requests to,
rather than sending actual network requests.
* **trust_env** - *(optional)* Enables or disables usage of environment
@ -972,6 +984,7 @@ class AsyncClient(BaseClient):
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
max_redirects: int = DEFAULT_MAX_REDIRECTS,
base_url: URLTypes = None,
transport: httpcore.AsyncHTTPTransport = None,
dispatch: httpcore.AsyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
@ -987,19 +1000,27 @@ class AsyncClient(BaseClient):
trust_env=trust_env,
)
if dispatch is not None:
warn_deprecated(
"The dispatch argument is deprecated since v0.13 and will be "
"removed in a future release, please use 'transport'",
)
if transport is None:
transport = dispatch
proxy_map = self.get_proxy_map(proxies, trust_env)
self.dispatch = self.init_dispatch(
self.transport = self.init_transport(
verify=verify,
cert=cert,
http2=http2,
pool_limits=pool_limits,
dispatch=dispatch,
transport=transport,
app=app,
trust_env=trust_env,
)
self.proxies: typing.Dict[str, httpcore.AsyncHTTPTransport] = {
key: self.init_proxy_dispatch(
key: self.init_proxy_transport(
proxy,
verify=verify,
cert=cert,
@ -1010,21 +1031,21 @@ class AsyncClient(BaseClient):
for key, proxy in proxy_map.items()
}
def init_dispatch(
def init_transport(
self,
verify: VerifyTypes = True,
cert: CertTypes = None,
http2: bool = False,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
dispatch: httpcore.AsyncHTTPTransport = None,
transport: httpcore.AsyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
) -> httpcore.AsyncHTTPTransport:
if dispatch is not None:
return dispatch
if transport is not None:
return transport
if app is not None:
return ASGIDispatch(app=app)
return ASGITransport(app=app)
ssl_context = SSLConfig(
verify=verify, cert=cert, trust_env=trust_env
@ -1039,7 +1060,7 @@ class AsyncClient(BaseClient):
http2=http2,
)
def init_proxy_dispatch(
def init_proxy_transport(
self,
proxy: Proxy,
verify: VerifyTypes = True,
@ -1064,7 +1085,7 @@ class AsyncClient(BaseClient):
http2=http2,
)
def dispatcher_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport:
def transport_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport:
"""
Returns the transport instance that should be used for a given URL.
This will either be the standard connection pool, or a proxy.
@ -1084,10 +1105,10 @@ class AsyncClient(BaseClient):
)
for proxy_key in proxy_keys:
if proxy_key and proxy_key in self.proxies:
dispatcher = self.proxies[proxy_key]
return dispatcher
transport = self.proxies[proxy_key]
return transport
return self.dispatch
return self.transport
async def request(
self,
@ -1222,7 +1243,7 @@ class AsyncClient(BaseClient):
Sends a single request, without handling any redirections.
"""
dispatcher = self.dispatcher_for_url(request.url)
transport = self.transport_for_url(request.url)
try:
(
@ -1231,7 +1252,7 @@ class AsyncClient(BaseClient):
reason_phrase,
headers,
stream,
) = await dispatcher.request(
) = await transport.request(
request.method.encode(),
request.url.raw,
headers=request.headers.raw,
@ -1434,7 +1455,7 @@ class AsyncClient(BaseClient):
)
async def aclose(self) -> None:
await self.dispatch.aclose()
await self.transport.aclose()
for proxy in self.proxies.values():
await proxy.aclose()

8
httpx/_models.py
View File

@ -4,7 +4,6 @@ import email.message
import json as jsonlib
import typing
import urllib.request
import warnings
from collections.abc import MutableMapping
from http.cookiejar import Cookie, CookieJar
from urllib.parse import parse_qsl, urlencode
@ -52,6 +51,7 @@ from ._utils import (
obfuscate_sensitive_headers,
parse_header_links,
str_query_param,
warn_deprecated,
)
if typing.TYPE_CHECKING: # pragma: no cover
@ -874,19 +874,17 @@ class Response:
@property
def stream(self): # type: ignore
warnings.warn( # pragma: nocover
warn_deprecated( # pragma: nocover
"Response.stream() is due to be deprecated. "
"Use Response.aiter_bytes() instead.",
DeprecationWarning,
)
return self.aiter_bytes # pragma: nocover
@property
def raw(self): # type: ignore
warnings.warn( # pragma: nocover
warn_deprecated( # pragma: nocover
"Response.raw() is due to be deprecated. "
"Use Response.aiter_raw() instead.",
DeprecationWarning,
)
return self.aiter_raw # pragma: nocover

0
httpx/_dispatch/__init__.py → httpx/_transports/__init__.py
View File

26
httpx/_dispatch/asgi.py → httpx/_transports/asgi.py
View File

@ -5,6 +5,7 @@ import httpcore
import sniffio
from .._content_streams import ByteStream
from .._utils import warn_deprecated
if typing.TYPE_CHECKING: # pragma: no cover
import asyncio
@ -24,9 +25,9 @@ def create_event() -> "Event":
return asyncio.Event()
class ASGIDispatch(httpcore.AsyncHTTPTransport):
class ASGITransport(httpcore.AsyncHTTPTransport):
"""
A custom AsyncDispatcher that handles sending requests directly to an ASGI app.
A custom AsyncTransport that handles sending requests directly to an ASGI app.
The simplest way to use this functionality is to use the `app` argument.
```
@ -35,10 +36,10 @@ class ASGIDispatch(httpcore.AsyncHTTPTransport):
Alternatively, you can setup the dispatch instance explicitly.
This allows you to include any additional configuration arguments specific
to the ASGIDispatch class:
to the ASGITransport class:
```
dispatch = httpx.ASGIDispatch(
dispatch = httpx.ASGITransport(
app=app,
root_path="/submount",
client=("1.2.3.4", 123)
@ -153,3 +154,20 @@ class ASGIDispatch(httpcore.AsyncHTTPTransport):
stream = ByteStream(b"".join(body_parts))
return (b"HTTP/1.1", status_code, b"", response_headers, stream)
class ASGIDispatch(ASGITransport):
def __init__(
self,
app: Callable,
raise_app_exceptions: bool = True,
root_path: str = "",
client: Tuple[str, int] = ("127.0.0.1", 123),
) -> None:
warn_deprecated("ASGIDispatch is deprecated, please use ASGITransport")
super().__init__(
app=app,
raise_app_exceptions=raise_app_exceptions,
root_path=root_path,
client=client,
)

36
httpx/_dispatch/urllib3.py → httpx/_transports/urllib3.py
View File

@ -4,16 +4,20 @@ import ssl
from typing import Dict, Iterator, List, Optional, Tuple, Union
import httpcore
import urllib3
from urllib3.exceptions import MaxRetryError, SSLError
from .._config import DEFAULT_POOL_LIMITS, PoolLimits, Proxy, SSLConfig
from .._content_streams import ByteStream, IteratorStream
from .._types import CertTypes, VerifyTypes
from .._utils import as_network_error
from .._utils import as_network_error, warn_deprecated
try:
import urllib3
from urllib3.exceptions import MaxRetryError, SSLError
except ImportError: # pragma: nocover
urllib3 = None
class URLLib3Dispatcher(httpcore.SyncHTTPTransport):
class URLLib3Transport(httpcore.SyncHTTPTransport):
def __init__(
self,
*,
@ -23,6 +27,10 @@ class URLLib3Dispatcher(httpcore.SyncHTTPTransport):
trust_env: bool = None,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
):
assert (
urllib3 is not None
), "urllib3 must be installed separately in order to use URLLib3Transport"
ssl_config = SSLConfig(
verify=verify, cert=cert, trust_env=trust_env, http2=False
)
@ -153,3 +161,23 @@ class URLLib3Dispatcher(httpcore.SyncHTTPTransport):
def close(self) -> None:
self.pool.clear()
class URLLib3Dispatch(URLLib3Transport):
def __init__(
self,
*,
proxy: Proxy = None,
verify: VerifyTypes = True,
cert: CertTypes = None,
trust_env: bool = None,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
):
warn_deprecated("URLLib3Dispatch is deprecated, please use URLLib3Transport")
super().__init__(
proxy=proxy,
verify=verify,
cert=cert,
trust_env=trust_env,
pool_limits=pool_limits,
)

24
httpx/_dispatch/wsgi.py → httpx/_transports/wsgi.py
View File

@ -5,6 +5,7 @@ import typing
import httpcore
from .._content_streams import ByteStream, IteratorStream
from .._utils import warn_deprecated
def _skip_leading_empty_chunks(body: typing.Iterable) -> typing.Iterable:
@ -15,7 +16,7 @@ def _skip_leading_empty_chunks(body: typing.Iterable) -> typing.Iterable:
return []
class WSGIDispatch(httpcore.SyncHTTPTransport):
class WSGITransport(httpcore.SyncHTTPTransport):
"""
A custom transport that handles sending requests directly to an WSGI app.
The simplest way to use this functionality is to use the `app` argument.
@ -26,10 +27,10 @@ class WSGIDispatch(httpcore.SyncHTTPTransport):
Alternatively, you can setup the dispatch instance explicitly.
This allows you to include any additional configuration arguments specific
to the WSGIDispatch class:
to the WSGITransport class:
```
dispatch = httpx.WSGIDispatch(
dispatch = httpx.WSGITransport(
app=app,
script_name="/submount",
remote_addr="1.2.3.4"
@ -131,3 +132,20 @@ class WSGIDispatch(httpcore.SyncHTTPTransport):
stream = IteratorStream(chunk for chunk in result)
return (b"HTTP/1.1", status_code, b"", headers, stream)
class WSGIDispatch(WSGITransport):
def __init__(
self,
app: typing.Callable,
raise_app_exceptions: bool = True,
script_name: str = "",
remote_addr: str = "127.0.0.1",
) -> None:
warn_deprecated("WSGIDispatch is deprecated, please use WSGITransport")
super().__init__(
app=app,
raise_app_exceptions=raise_app_exceptions,
script_name=script_name,
remote_addr=remote_addr,
)

5
httpx/_utils.py
View File

@ -8,6 +8,7 @@ import os
import re
import sys
import typing
import warnings
from datetime import timedelta
from pathlib import Path
from time import perf_counter
@ -400,3 +401,7 @@ def as_network_error(*exception_classes: type) -> typing.Iterator[None]:
if isinstance(exc, cls):
raise NetworkError(exc) from exc
raise
def warn_deprecated(message: str) -> None:
warnings.warn(message, DeprecationWarning)

2
tests/client/test_auth.py
View File

@ -484,7 +484,7 @@ async def test_digest_auth_unavailable_streaming_body():
client = AsyncClient(dispatch=MockDispatch())
async def streaming_body():
yield b"Example request body"
yield b"Example request body" # pragma: nocover
with pytest.raises(RequestBodyUnavailable):
await client.post(url, data=streaming_body(), auth=auth)

14
tests/client/test_proxies.py
View File

@ -74,14 +74,14 @@ PROXY_URL = "http://[::1]"
),
],
)
def test_dispatcher_for_request(url, proxies, expected):
def test_transport_for_request(url, proxies, expected):
client = httpx.AsyncClient(proxies=proxies)
dispatcher = client.dispatcher_for_url(httpx.URL(url))
transport = client.transport_for_url(httpx.URL(url))
if expected is None:
assert dispatcher is client.dispatch
assert transport is client.transport
else:
assert dispatcher.proxy_origin == httpx.URL(expected).raw[:3]
assert transport.proxy_origin == httpx.URL(expected).raw[:3]
def test_unsupported_proxy_scheme():
@ -110,9 +110,9 @@ def test_proxies_environ(monkeypatch, url, env, expected):
monkeypatch.setenv(name, value)
client = httpx.AsyncClient()
dispatcher = client.dispatcher_for_url(httpx.URL(url))
transport = client.transport_for_url(httpx.URL(url))
if expected is None:
assert dispatcher == client.dispatch
assert transport == client.transport
else:
assert dispatcher.proxy_origin == httpx.URL(expected).raw[:3]
assert transport.proxy_origin == httpx.URL(expected).raw[:3]