I am using pyppeteer to take a screenshot https://pypi.org/project/pyppeteer/. But it takes a lot of time so wanted to use multiprocessing. This is my code.
import nest_asyncio
nest_asyncio.apply()
import asyncio
from pyppeteer import launch
urls_list=['https://pypi.org/project/pyppeteer/','https://www.google.com/','https://stackoverflow.com/']
async def screenshot(url):
browser = await launch()
page = await browser.newPage()
await page.goto('https://example.com')
await page.screenshot({'path': 'example.png'})
await browser.close()
asyncio.get_event_loop().run_until_complete(main())
from multiprocessing.dummy import Pool
from multiprocessing import cpu_count
threads=cpu_count()*2
p = Pool(threads) # Pool tells how many at a time
p.map(asyncio.get_event_loop().run_until_complete(screenshot),urls_list)
p.terminate()
p.join()
And I get the following error. How do I use all my cores to run this async function?
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-56-8b71a3dbc919> in <module>
18 p = Pool(threads) # Pool tells how many at a time
19 # rows = p.map(asyncio.get_event_loop().run_until_complete(screenshot),urls_subset)
---> 20 rows = p.apply_async(asyncio.get_event_loop().run_until_complete(screenshot),urls_subset)
21
22 p.terminate()
~/opt/anaconda3/envs/dlcpu/lib/python3.8/site-packages/nest_asyncio.py in run_until_complete(self, future)
83 self._check_closed()
84 events._set_running_loop(self)
---> 85 f = asyncio.ensure_future(future, loop=self)
86 if f is not future:
87 f._log_destroy_pending = False
~/opt/anaconda3/envs/dlcpu/lib/python3.8/asyncio/tasks.py in ensure_future(coro_or_future, loop)
671 return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
672 else:
--> 673 raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
674 'required')
675
TypeError: An asyncio.Future, a coroutine or an awaitable is required