encode/httpx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Response.content

Browse Source
This commit is contained in:
Tom Christie 2019-04-30 14:58:43 +01:00
parent dea4d42c6d
commit 4c5511313c
14 changed files with 130 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
httpcore/__init__.py
View File

@ -14,6 +14,7 @@ from .exceptions import (
RedirectBodyUnavailable,
RedirectLoop,
ResponseClosed,
ResponseNotRead,
StreamConsumed,
Timeout,
TooManyRedirects,

2
httpcore/dispatch/http11.py
View File

@ -85,7 +85,7 @@ class HTTP11Connection(Adapter):
reason_phrase=reason_phrase,
protocol="HTTP/1.1",
headers=headers,
body=body,
content=body,
on_close=self.response_closed,
request=request,
)

2
httpcore/dispatch/http2.py
View File

@ -73,7 +73,7 @@ class HTTP2Connection(Adapter):
status_code=status_code,
protocol="HTTP/2",
headers=headers,
body=body,
content=body,
on_close=on_close,
request=request,
)

10
httpcore/exceptions.py
View File

@ -78,6 +78,13 @@ class StreamConsumed(Exception):
"""
class ResponseNotRead(Exception):
"""
Attempted to access response content, without having called `read()`
after a streaming response.
"""
class ResponseClosed(Exception):
"""
Attempted to read or stream response content, but the request has been
@ -91,6 +98,9 @@ class DecodingError(Exception):
"""
# Other cases...
class InvalidURL(Exception):
"""
URL was missing a hostname, or was not one of HTTP/HTTPS.

85
httpcore/models.py
View File

@ -9,7 +9,7 @@ from .decoders import (
IdentityDecoder,
MultiDecoder,
)
from .exceptions import ResponseClosed, StreamConsumed
from .exceptions import ResponseClosed, ResponseNotRead, StreamConsumed
from .status_codes import codes
from .utils import get_reason_phrase, normalize_header_key, normalize_header_value
@ -197,7 +197,9 @@ class Headers(typing.MutableMapping[str, str]):
except KeyError:
return default
def getlist(self, key: str, default: typing.Any = None, split_commas = None) -> typing.List[str]:
def getlist(
self, key: str, split_commas: bool=False
) -> typing.List[str]:
"""
Return multiple header values.
@ -208,7 +210,7 @@ class Headers(typing.MutableMapping[str, str]):
"""
get_header_key = key.lower().encode(self.encoding)
if split_commas is None:
split_commas = get_header_key != b'set-cookie'
split_commas = get_header_key != b"set-cookie"
values = [
item_value.decode(self.encoding)
@ -216,9 +218,6 @@ class Headers(typing.MutableMapping[str, str]):
if item_key == get_header_key
]
if not values:
return [] if default is None else default
if not split_commas:
return values
@ -352,6 +351,13 @@ class Request:
yield self.body
def prepare(self) -> None:
"""
Adds in any default headers. When using the `Client`, this will
end up being called into by the `prepare_request()` stage.
You can omit this behavior by calling `Client.send()` with an
explicitly built `Request` instance.
"""
auto_headers = [] # type: typing.List[typing.Tuple[bytes, bytes]]
has_host = "host" in self.headers
@ -383,28 +389,26 @@ class Response:
reason_phrase: str = None,
protocol: str = None,
headers: HeaderTypes = None,
body: BodyTypes = b"",
content: BodyTypes = b"",
on_close: typing.Callable = None,
request: Request = None,
history: typing.List["Response"] = None,
):
self.status_code = status_code
if reason_phrase is None:
self.reason_phrase = get_reason_phrase(status_code)
else:
self.reason_phrase = reason_phrase
self.reason_phrase = reason_phrase or get_reason_phrase(status_code)
self.protocol = protocol
self.headers = Headers(headers)
self.on_close = on_close
self.is_closed = False
self.is_streamed = False
if isinstance(body, bytes):
if isinstance(content, bytes):
self.is_closed = True
self.body = self.decoder.decode(body) + self.decoder.flush()
self.is_stream_consumed = True
self._raw_content = content
else:
self.body_aiter = body
self.is_closed = False
self.is_stream_consumed = False
self._raw_stream = content
self.on_close = on_close
self.request = request
self.history = [] if history is None else list(history)
self.next = None # typing.Optional[typing.Callable]
@ -418,6 +422,17 @@ class Response:
"""
return None if self.request is None else self.request.url
@property
def content(self) -> bytes:
if not hasattr(self, "_content"):
if hasattr(self, "_raw_content"):
self._content = (
self.decoder.decode(self._raw_content) + self.decoder.flush()
)
else:
raise ResponseNotRead()
return self._content
@property
def decoder(self) -> Decoder:
"""
@ -426,7 +441,7 @@ class Response:
"""
if not hasattr(self, "_decoder"):
decoders = [] # type: typing.List[Decoder]
values = self.headers.getlist("content-encoding", ["identity"])
values = self.headers.getlist("content-encoding", split_commas=True)
for value in values:
value = value.strip().lower()
decoder_cls = SUPPORTED_DECODERS[value]
@ -445,20 +460,20 @@ class Response:
"""
Read and return the response content.
"""
if not hasattr(self, "body"):
body = b""
if not hasattr(self, "_content"):
content = b""
async for part in self.stream():
body += part
self.body = body
return self.body
content += part
self._content = content
return self._content
async def stream(self) -> typing.AsyncIterator[bytes]:
"""
A byte-iterator over the decoded response content.
This allows us to handle gzip, deflate, and brotli encoded responses.
"""
if hasattr(self, "body"):
yield self.body
if hasattr(self, "_content"):
yield self._content
else:
async for chunk in self.raw():
yield self.decoder.decode(chunk)
@ -468,14 +483,18 @@ class Response:
"""
A byte-iterator over the raw response content.
"""
if self.is_streamed:
raise StreamConsumed()
if self.is_closed:
raise ResponseClosed()
self.is_streamed = True
async for part in self.body_aiter:
yield part
await self.close()
if hasattr(self, "_raw_content"):
yield self._raw_content
else:
if self.is_stream_consumed:
raise StreamConsumed()
if self.is_closed:
raise ResponseClosed()
self.is_stream_consumed = True
async for part in self._raw_stream:
yield part
await self.close()
async def close(self) -> None:
"""

4
httpcore/sync.py
View File

@ -26,8 +26,8 @@ class SyncResponse:
return self._response.headers
@property
def body(self) -> bytes:
return self._response.body
def content(self) -> bytes:
return self._response.content
def read(self) -> bytes:
return self._loop.run_until_complete(self._response.read())

12
tests/adapters/test_redirects.py
View File

@ -66,7 +66,7 @@ class MockDispatch(Adapter):
elif request.url.path == "/cross_domain_target":
headers = dict(request.headers.items())
body = json.dumps({"headers": headers}).encode()
return Response(codes.ok, body=body, request=request)
return Response(codes.ok, content=body, request=request)
elif request.url.path == "/redirect_body":
body = await request.read()
@ -76,9 +76,9 @@ class MockDispatch(Adapter):
elif request.url.path == "/redirect_body_target":
body = await request.read()
body = json.dumps({"body": body.decode()}).encode()
return Response(codes.ok, body=body, request=request)
return Response(codes.ok, content=body, request=request)
return Response(codes.ok, body=b"Hello, world!", request=request)
return Response(codes.ok, content=b"Hello, world!", request=request)
@pytest.mark.asyncio
@ -202,7 +202,7 @@ async def test_cross_domain_redirect():
url = "https://example.com/cross_domain"
headers = {"Authorization": "abc"}
response = await client.request("GET", url, headers=headers)
data = json.loads(response.body.decode())
data = json.loads(response.content.decode())
assert response.url == URL("https://example.org/cross_domain_target")
assert data == {"headers": {}}
@ -213,7 +213,7 @@ async def test_same_domain_redirect():
url = "https://example.org/cross_domain"
headers = {"Authorization": "abc"}
response = await client.request("GET", url, headers=headers)
data = json.loads(response.body.decode())
data = json.loads(response.content.decode())
assert response.url == URL("https://example.org/cross_domain_target")
assert data == {"headers": {"authorization": "abc"}}
@ -224,7 +224,7 @@ async def test_body_redirect():
url = "https://example.org/redirect_body"
body = b"Example request body"
response = await client.request("POST", url, body=body)
data = json.loads(response.body.decode())
data = json.loads(response.content.decode())
assert response.url == URL("https://example.org/redirect_body_target")
assert data == {"body": "Example request body"}

2
tests/dispatch/test_connections.py
View File

@ -8,7 +8,7 @@ async def test_get(server):
http = httpcore.HTTPConnection(origin="http://127.0.0.1:8000/")
response = await http.request("GET", "http://127.0.0.1:8000/")
assert response.status_code == 200
assert response.body == b"Hello, world!"
assert response.content == b"Hello, world!"
@pytest.mark.asyncio

10
tests/dispatch/test_http2.py
View File

@ -82,7 +82,7 @@ async def test_http2_get_request():
async with httpcore.HTTP2Connection(reader=server, writer=server) as conn:
response = await conn.request("GET", "http://example.org")
assert response.status_code == 200
assert json.loads(response.body) == {"method": "GET", "path": "/", "body": ""}
assert json.loads(response.content) == {"method": "GET", "path": "/", "body": ""}
@pytest.mark.asyncio
@ -91,7 +91,7 @@ async def test_http2_post_request():
async with httpcore.HTTP2Connection(reader=server, writer=server) as conn:
response = await conn.request("POST", "http://example.org", body=b"<data>")
assert response.status_code == 200
assert json.loads(response.body) == {
assert json.loads(response.content) == {
"method": "POST",
"path": "/",
"body": "<data>",
@ -107,10 +107,10 @@ async def test_http2_multiple_requests():
response_3 = await conn.request("GET", "http://example.org/3")
assert response_1.status_code == 200
assert json.loads(response_1.body) == {"method": "GET", "path": "/1", "body": ""}
assert json.loads(response_1.content) == {"method": "GET", "path": "/1", "body": ""}
assert response_2.status_code == 200
assert json.loads(response_2.body) == {"method": "GET", "path": "/2", "body": ""}
assert json.loads(response_2.content) == {"method": "GET", "path": "/2", "body": ""}
assert response_3.status_code == 200
assert json.loads(response_3.body) == {"method": "GET", "path": "/3", "body": ""}
assert json.loads(response_3.content) == {"method": "GET", "path": "/3", "body": ""}

8
tests/models/test_headers.py
View File

@ -147,8 +147,8 @@ def test_multiple_headers():
"""
Most headers should split by commas for `getlist`, except 'Set-Cookie'.
"""
h = httpcore.Headers([('set-cookie', 'a, b'), ('set-cookie', 'c')])
h.getlist('Set-Cookie') == ['a, b', 'b']
h = httpcore.Headers([("set-cookie", "a, b"), ("set-cookie", "c")])
h.getlist("Set-Cookie") == ["a, b", "b"]
h = httpcore.Headers([('vary', 'a, b'), ('vary', 'c')])
h.getlist('Vary') == ['a', 'b', 'c']
h = httpcore.Headers([("vary", "a, b"), ("vary", "c")])
h.getlist("Vary") == ["a", "b", "c"]

31
tests/models/test_responses.py
View File

@ -9,50 +9,49 @@ async def streaming_body():
def test_response():
response = httpcore.Response(200, body=b"Hello, world!")
response = httpcore.Response(200, content=b"Hello, world!")
assert response.status_code == 200
assert response.reason_phrase == "OK"
assert response.body == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
@pytest.mark.asyncio
async def test_read_response():
response = httpcore.Response(200, body=b"Hello, world!")
response = httpcore.Response(200, content=b"Hello, world!")
assert response.status_code == 200
assert response.body == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
body = await response.read()
content = await response.read()
assert body == b"Hello, world!"
assert response.body == b"Hello, world!"
assert content == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
@pytest.mark.asyncio
async def test_streaming_response():
response = httpcore.Response(200, body=streaming_body())
response = httpcore.Response(200, content=streaming_body())
assert response.status_code == 200
assert not hasattr(response, "body")
assert not response.is_closed
body = await response.read()
content = await response.read()
assert body == b"Hello, world!"
assert response.body == b"Hello, world!"
assert content == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
@pytest.mark.asyncio
async def test_cannot_read_after_stream_consumed():
response = httpcore.Response(200, body=streaming_body())
response = httpcore.Response(200, content=streaming_body())
body = b""
content = b""
async for part in response.stream():
body += part
content += part
with pytest.raises(httpcore.StreamConsumed):
await response.read()
@ -60,7 +59,7 @@ async def test_cannot_read_after_stream_consumed():
@pytest.mark.asyncio
async def test_cannot_read_after_response_closed():
response = httpcore.Response(200, body=streaming_body())
response = httpcore.Response(200, content=streaming_body())
await response.close()

13
tests/test_client.py
View File

@ -9,7 +9,7 @@ async def test_get(server):
async with httpcore.Client() as client:
response = await client.get(url)
assert response.status_code == 200
assert response.body == b"Hello, world!"
assert response.content == b"Hello, world!"
@pytest.mark.asyncio
@ -25,9 +25,18 @@ async def test_stream_response(server):
async with httpcore.Client() as client:
response = await client.request("GET", "http://127.0.0.1:8000/", stream=True)
assert response.status_code == 200
assert not hasattr(response, "body")
body = await response.read()
assert body == b"Hello, world!"
assert response.content == b"Hello, world!"
@pytest.mark.asyncio
async def test_access_content_stream_response(server):
async with httpcore.Client() as client:
response = await client.request("GET", "http://127.0.0.1:8000/", stream=True)
assert response.status_code == 200
with pytest.raises(httpcore.ResponseNotRead):
response.content
@pytest.mark.asyncio

29
tests/test_decoders.py
View File

@ -12,8 +12,8 @@ def test_deflate():
compressed_body = compressor.compress(body) + compressor.flush()
headers = [(b"Content-Encoding", b"deflate")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
def test_gzip():
@ -22,8 +22,8 @@ def test_gzip():
compressed_body = compressor.compress(body) + compressor.flush()
headers = [(b"Content-Encoding", b"gzip")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
def test_brotli():
@ -31,8 +31,8 @@ def test_brotli():
compressed_body = brotli.compress(body)
headers = [(b"Content-Encoding", b"br")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
def test_multi():
@ -47,8 +47,8 @@ def test_multi():
)
headers = [(b"Content-Encoding", b"deflate, gzip")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
def test_multi_with_identity():
@ -56,12 +56,12 @@ def test_multi_with_identity():
compressed_body = brotli.compress(body)
headers = [(b"Content-Encoding", b"br, identity")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
headers = [(b"Content-Encoding", b"identity, br")]
response = httpcore.Response(200, headers=headers, body=compressed_body)
assert response.body == body
response = httpcore.Response(200, headers=headers, content=compressed_body)
assert response.content == body
@pytest.mark.asyncio
@ -74,7 +74,7 @@ async def test_streaming():
yield compressor.flush()
headers = [(b"Content-Encoding", b"gzip")]
response = httpcore.Response(200, headers=headers, body=compress(body))
response = httpcore.Response(200, headers=headers, content=compress(body))
assert not hasattr(response, "body")
assert await response.read() == body
@ -85,4 +85,5 @@ def test_decoding_errors(header_value):
body = b"test 123"
compressed_body = brotli.compress(body)[3:]
with pytest.raises(httpcore.exceptions.DecodingError):
response = httpcore.Response(200, headers=headers, body=compressed_body)
response = httpcore.Response(200, headers=headers, content=compressed_body)
response.content

13
tests/test_sync.py
View File

@ -28,7 +28,7 @@ def test_get(server):
with httpcore.SyncConnectionPool() as http:
response = http.request("GET", "http://127.0.0.1:8000/")
assert response.status_code == 200
assert response.body == b"Hello, world!"
assert response.content == b"Hello, world!"
@threadpool
@ -43,9 +43,8 @@ def test_stream_response(server):
with httpcore.SyncConnectionPool() as http:
response = http.request("GET", "http://127.0.0.1:8000/", stream=True)
assert response.status_code == 200
assert not hasattr(response, "body")
body = response.read()
assert body == b"Hello, world!"
content = response.read()
assert content == b"Hello, world!"
@threadpool
@ -53,7 +52,7 @@ def test_stream_iterator(server):
with httpcore.SyncConnectionPool() as http:
response = http.request("GET", "http://127.0.0.1:8000/", stream=True)
assert response.status_code == 200
body = b""
content = b""
for chunk in response.stream():
body += chunk
assert body == b"Hello, world!"
content += chunk
assert content == b"Hello, world!"