how to use async functions with multiprocessing in python?

338 views Asked by At

I am using pyppeteer to take a screenshot https://pypi.org/project/pyppeteer/. But it takes a lot of time so wanted to use multiprocessing. This is my code.

import nest_asyncio
nest_asyncio.apply()
import asyncio
from pyppeteer import launch
urls_list=['https://pypi.org/project/pyppeteer/','https://www.google.com/','https://stackoverflow.com/']
async def screenshot(url):
    browser = await launch()
    page = await browser.newPage()
    await page.goto('https://example.com')
    await page.screenshot({'path': 'example.png'})
    await browser.close()

asyncio.get_event_loop().run_until_complete(main())

from multiprocessing.dummy import Pool
from multiprocessing import cpu_count
threads=cpu_count()*2

p = Pool(threads)  # Pool tells how many at a time
p.map(asyncio.get_event_loop().run_until_complete(screenshot),urls_list)
p.terminate()
p.join()

And I get the following error. How do I use all my cores to run this async function?

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-56-8b71a3dbc919> in <module>
     18     p = Pool(threads)  # Pool tells how many at a time
     19 #     rows = p.map(asyncio.get_event_loop().run_until_complete(screenshot),urls_subset)
---> 20     rows = p.apply_async(asyncio.get_event_loop().run_until_complete(screenshot),urls_subset)
     21 
     22     p.terminate()

~/opt/anaconda3/envs/dlcpu/lib/python3.8/site-packages/nest_asyncio.py in run_until_complete(self, future)
     83             self._check_closed()
     84             events._set_running_loop(self)
---> 85             f = asyncio.ensure_future(future, loop=self)
     86             if f is not future:
     87                 f._log_destroy_pending = False

~/opt/anaconda3/envs/dlcpu/lib/python3.8/asyncio/tasks.py in ensure_future(coro_or_future, loop)
    671         return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
    672     else:
--> 673         raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
    674                         'required')
    675 

TypeError: An asyncio.Future, a coroutine or an awaitable is required
0

There are 0 answers