FastAPI + aioboto3: How to implement iter_chunks?

548 views Asked by At

I'd like to download file from S3 and return it in API endpoint. For this purpose I developed simple FastAPI app:

from io import BytesIO

import aioboto3
import uvicorn
from fastapi import FastAPI
from starlette.responses import StreamingResponse

app = FastAPI()


async def download_file(key):
    session = aioboto3.Session(aws_access_key_id='xxx', aws_secret_access_key='yyy')
    async with session.client("s3", endpoint_url='http://localhost:9000') as s3:
        return await s3.get_object(Bucket='my-backups-12345', Key=key)


@app.get('/works/')
async def works():
    # data being downloaded
    key = 'facd0b48-312b-4cf8-8513-7ad15a8f7bae/2-2.xlsx'

    session = aioboto3.Session(aws_access_key_id='xxx',
                              aws_secret_access_key='yyy')
    async with session.client("s3", endpoint_url='http://localhost:9000') as s3:
        resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
        content = await resp['Body'].read()

        return StreamingResponse(BytesIO(content))


@app.get('/does-not-work/')
async def does_not_work():
    # data are not being downloaded
    # how to implement iter_chunks

    key = 'facd0b48-312b-4cf8-8513-7ad15a8f7bae/2-2.xlsx'
    session = aioboto3.Session(aws_access_key_id='tlBne2gPKidWDkKx',
                               aws_secret_access_key='p8ClBh5h0nsjb98MeDs6k5l5gI0OZ1bB')

    async with session.client("s3", use_ssl=False, endpoint_url='http://localhost:9000', verify=False) as s3:
        resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
        return StreamingResponse(resp['Body'].iter_chunks())


if __name__ == '__main__':
    uvicorn.run(app, host="0.0.0.0")

The problem with API does-not-work. Is the way I'm trying to implement iter_chunks right? I'd like to expose chuck at the moment I get it from S3 if possible of course.

UPD. I guess there is something with the last chunk https://github.com/encode/starlette/blob/master/starlette/responses.py#L247

UPD2. As far as I can there is something wrong with aioboto3 session/client because this works as expected:

import aioboto3
from starlette.responses import FileResponse, StreamingResponse


class S3FileStreamingResponse(StreamingResponse):
    """Represent an FS File Response."""

    chunk_size = 4096

    async def __call__(self, scope, receive, send) -> None:
        """Make the call."""
        key = '0d8bba25-4fae-4e5f-8804-4c9282a193cb/2-2.xlsx'
        session = aioboto3.Session(aws_access_key_id='tlBne2gPKidWDkKx',
                                   aws_secret_access_key='p8ClBh5h0nsjb98MeDs6k5l5gI0OZ1bB')

        async with session.client("s3", use_ssl=False, endpoint_url='http://localhost:9000', verify=False) as s3:
            resp = await s3.get_object(Bucket='my-backups-12345', Key=key)
            time.sleep(10)
            total_size = resp["ContentLength"]
            sent_size = 0

            await send(
                {
                    "type": "http.response.start",
                    "status": self.status_code,
                    "headers": self.raw_headers,
                }
            )

            async for chunk in resp["Body"].iter_chunks():
                sent_size += len(chunk)
                await send(
                    {
                        "type": "http.response.body",
                        "body": chunk,
                        "more_body": sent_size < total_size,
                    }
                )

        if self.background is not None:
            await self.background()
0

There are 0 answers