I'd like to download file from S3 and return it in API endpoint. For this purpose I developed simple FastAPI app:
from io import BytesIO
import aioboto3
import uvicorn
from fastapi import FastAPI
from starlette.responses import StreamingResponse
app = FastAPI()
async def download_file(key):
session = aioboto3.Session(aws_access_key_id='xxx', aws_secret_access_key='yyy')
async with session.client("s3", endpoint_url='http://localhost:9000') as s3:
return await s3.get_object(Bucket='my-backups-12345', Key=key)
@app.get('/works/')
async def works():
# data being downloaded
key = 'facd0b48-312b-4cf8-8513-7ad15a8f7bae/2-2.xlsx'
session = aioboto3.Session(aws_access_key_id='xxx',
aws_secret_access_key='yyy')
async with session.client("s3", endpoint_url='http://localhost:9000') as s3:
resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
content = await resp['Body'].read()
return StreamingResponse(BytesIO(content))
@app.get('/does-not-work/')
async def does_not_work():
# data are not being downloaded
# how to implement iter_chunks
key = 'facd0b48-312b-4cf8-8513-7ad15a8f7bae/2-2.xlsx'
session = aioboto3.Session(aws_access_key_id='tlBne2gPKidWDkKx',
aws_secret_access_key='p8ClBh5h0nsjb98MeDs6k5l5gI0OZ1bB')
async with session.client("s3", use_ssl=False, endpoint_url='http://localhost:9000', verify=False) as s3:
resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
return StreamingResponse(resp['Body'].iter_chunks())
if __name__ == '__main__':
uvicorn.run(app, host="0.0.0.0")
The problem with API does-not-work
. Is the way I'm trying to implement iter_chunks
right? I'd like to expose chuck at the moment I get it from S3 if possible of course.
UPD. I guess there is something with the last chunk https://github.com/encode/starlette/blob/master/starlette/responses.py#L247
UPD2. As far as I can there is something wrong with aioboto3 session/client because this works as expected:
import aioboto3
from starlette.responses import FileResponse, StreamingResponse
class S3FileStreamingResponse(StreamingResponse):
"""Represent an FS File Response."""
chunk_size = 4096
async def __call__(self, scope, receive, send) -> None:
"""Make the call."""
key = '0d8bba25-4fae-4e5f-8804-4c9282a193cb/2-2.xlsx'
session = aioboto3.Session(aws_access_key_id='tlBne2gPKidWDkKx',
aws_secret_access_key='p8ClBh5h0nsjb98MeDs6k5l5gI0OZ1bB')
async with session.client("s3", use_ssl=False, endpoint_url='http://localhost:9000', verify=False) as s3:
resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
time.sleep(10)
total_size = resp["ContentLength"]
sent_size = 0
await send(
{
"type": "http.response.start",
"status": self.status_code,
"headers": self.raw_headers,
}
)
async for chunk in resp["Body"].iter_chunks():
sent_size += len(chunk)
await send(
{
"type": "http.response.body",
"body": chunk,
"more_body": sent_size < total_size,
}
)
if self.background is not None:
await self.background()