Too many open files when using multiprocessing

26 views Asked by At

I have written this code (discover/nodes.py) which get me list of nodes that have specific port (let's say 8134) open.

import os
import multiprocessing
import socket


# Local imports
from . import scan
from .. import TCP_PORT

LOCAL_DNS = "192.168.1.1"


def pinger(job_q, results_q):
    """
    Do Ping
    :param job_q:
    :param results_q:
    :return:
    """
    while True:

        ip = job_q.get()
        if ip is None:
            break

        try:
            result = scan.is_port_open(ip, TCP_PORT)
            if result:
                results_q.put(ip)
        except:
            pass
    return False


def get_my_ip():
    """
    Find my IP address
    :return:
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect((LOCAL_DNS, 80))
    ip = s.getsockname()[0]
    s.close()
    return ip


def map_network(start=1, pool_size=255):
    """
    Maps the network
    :param pool_size: amount of parallel ping processes
    :return: list of valid ip addresses
    """
    for key in [start, pool_size]:
        if not isinstance(key, int):
            raise ValueError("{0} is not of type int.".format(key))

    ip_list = list()

    # get my IP and compose a base like 192.168.1.xxx
    ip_parts = get_my_ip().split('.')
    base_ip = ip_parts[0] + '.' + ip_parts[1] + '.' + ip_parts[2] + '.'

    # prepare the jobs queue
    jobs = multiprocessing.Queue()
    results = multiprocessing.Queue()

    pool = [
        multiprocessing.Process(
            target=pinger, args=(jobs, results)
        ) for i in range(pool_size)
    ]

    for p in pool:
        p.start()

    # queue the ping processes
    for i in range(start, pool_size):
        jobs.put(base_ip + '{0}'.format(i))

    for p in pool:
        jobs.put(None)

    for p in pool:
        p.join()

    # collect he results
    while not results.empty():
        ip = results.get()
        ip_list.append(ip)

    return ip_list

This works in Ubuntu on Python 3.10 however I am trying to make it work on an old Mac OS running Catalina 10.15.7 using Python 3.9.

But this is resulting in error:

python
Python 3.9.6 (default, Jun 29 2021, 06:20:32) 
[Clang 12.0.0 (clang-1200.0.32.29)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> from discover.nodes import map_network
>>> ips = map_network()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/ciasto/Downloads/discovery-main-2/.venv/lib/python3.9/site-packages/discovery/nodes.py", line 74, in map_network
    p.start()
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 58, in _launch
    self.pid = util.spawnv_passfds(spawn.get_executable(),
  File "/usr/local/Cellar/[email protected]/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/util.py", line 450, in spawnv_passfds
    errpipe_read, errpipe_write = os.pipe()
OSError: [Errno 24] Too many open files

What is it referring to too many files open? Is there an alternative to using multiprocessing you would recommend?

0

There are 0 answers