Keras run 2 independent training process on 2 GPU. ResourceExhaustedError

Question

I have 2 GPU GTX1080 with Keras v2 installed. I run 2 training processes on gpu0 and gpu1 simultaneously. But, I got ResourceExhaustedError.

What is missing?

python multi-gpu-process.py --gpu_id=1 (ok)

python multi-gpu-process.py --gpu_id=0 (ResourceExhaustedError. Help pls.)

# file: multi-gpu-process.py  (2 training processes work on different GPUs)

    import numpy as np
    import os



    def get_available_gpus():
      local_device_protos = device_lib.list_local_devices()
      return [x.name for x in local_device_protos if x.device_type == 'GPU']

    # What GPU is installed.
    gpu_list = get_available_gpus()

    # Specified gpu installed on machine?
    if not '/gpu:' + str(FLAGS.gpu_id) in gpu_list:
      raise Exception('This gpu is not installed: /gpu:{}'.format(FLAGS.gpu_id))

    # Set GPU in environment.
    os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' # see issue #152
    os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)



    import keras
    from keras.models import Sequential
    from keras.layers import Flatten, Dense, Dropout, Activation
    from keras.layers.convolutional import Conv2D
    from keras.layers.normalization import BatchNormalization
    from keras.optimizers import Adam
    from keras.callbacks import ModelCheckpoint

    import tensorflow as tf
    from tensorflow.python.client import device_lib

    FLAGS = tf.app.flags.FLAGS

    tf.app.flags.DEFINE_integer('gpu_id', 0, """GPU id. Single gpu case.""")

    # Use one GPU.
    import keras.backend.tensorflow_backend as K   # If import and not used, error on session release.


    # Solve this error
    # https://stackoverflow.com/questions/42969779/keras-error-you-must-feed-a-value-for-placeholder-tensor-bidirectional-1-keras
    K.set_learning_phase(1) # set learning phase

    # train on specified gpu
    with K.tf.device('/gpu:%d' % FLAGS.gpu_id):
      K.set_session(K.tf.Session(config=K.tf.ConfigProto(allow_soft_placement=True,  # True. Allow to find other device if specified is not available.
                                                 log_device_placement=True)))
      # To prove running multi process on gpu. Make small model.
      model = Sequential()
      model.add(Dense(400, input_dim=800, activation='tanh'))
      model.add(Dense(200, input_dim=800, activation='relu'))
      model.add(Dense(50, activation='relu'))
      model.add(Dense(30, activation='relu'))
      model.add(Dense(1, activation='sigmoid'))

      print (model.summary())

      optimizer = keras.optimizers.Adam(lr=0.0001)
      model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])

      x = np.random.rand(131072, 800)
      y = np.random.randint(0, 2, (131072, 1))

      model.fit(x, y, batch_size=2048*4, epochs=1000000)

    K.clear_session() # Without it, session error at the end.



    ==================Not work when starting 2nd process on 2nd GPU (GTX 1080, 8GB)=========================
    # Increased model size compared with working version
    #
    # file: multi-gpu-process_notwork.py  

    import numpy as np
    import os



    def get_available_gpus():
      local_device_protos = device_lib.list_local_devices()
      return [x.name for x in local_device_protos if x.device_type == 'GPU']

    # What GPU is installed.
    gpu_list = get_available_gpus()

    # Specified gpu installed on machine?
    if not '/gpu:' + str(FLAGS.gpu_id) in gpu_list:
      raise Exception('This gpu is not installed: /gpu:{}'.format(FLAGS.gpu_id))

    # Set GPU in environment.
    os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' # see issue #152
    os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)



    import keras
    from keras.models import Sequential
    from keras.layers import Flatten, Dense, Dropout, Activation
    from keras.layers.convolutional import Conv2D
    from keras.layers.normalization import BatchNormalization
    from keras.optimizers import Adam
    from keras.callbacks import ModelCheckpoint

    import tensorflow as tf
    from tensorflow.python.client import device_lib

    FLAGS = tf.app.flags.FLAGS

    tf.app.flags.DEFINE_integer('gpu_id', 0, """GPU id. Single gpu case.""")

    # Use one GPU.
    import keras.backend.tensorflow_backend as K   # If import and not used, error on session release.


    # Solve this error
    # https://stackoverflow.com/questions/42969779/keras-error-you-must-feed-a-value-for-placeholder-tensor-bidirectional-1-keras
    K.set_learning_phase(1) # set learning phase

    # train on specified gpu
    with K.tf.device('/gpu:%d' % FLAGS.gpu_id):
      K.set_session(K.tf.Session(config=K.tf.ConfigProto(allow_soft_placement=True,  # True. Allow to find other device if specified is not available.
                                                 log_device_placement=True)))
      # To prove running multi process on gpu. Make small model.
      model = Sequential()
      model.add(Dense(4000, input_dim=8000, activation='tanh'))
      model.add(Dense(2000, input_dim=8000, activation='relu'))
      model.add(Dense(50, activation='relu'))
      model.add(Dense(30, activation='relu'))
      model.add(Dense(1, activation='sigmoid'))

      print (model.summary())

      optimizer = keras.optimizers.Adam(lr=0.0001)
      model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])

      x = np.random.rand(131072, 8000)
      y = np.random.randint(0, 2, (131072, 1))

      model.fit(x, y, batch_size=2048*4, epochs=1000000)

    K.clear_session() # Without it, session error at the end.

Partial error:

ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[8192,400]
   [[Node: gradients/dense_1/Tanh_grad/TanhGrad = TanhGrad[T=DT_FLOAT, _class=["loc:@dense_1/Tanh"], _device="/job:localhost/replica:0/task:0/gpu:0"](dense_1/Tanh, gradients/dense_2/MatMul_grad/MatMul)]]

Since the GPU has 8GB, it has nothing to do with model, which is pretty small size.

Answer 1

Your model may be small, but your batch size of 8192 is probably way too big for your memory; start with a value of 64 or 128, and then you can try increasing it until you get an out-of-memory (OOM) error again...

EDIT: Although reducing your batch size will probably eliminate the error, there is still the issue of discrepancy between your 2 GPUs - as Yu-Yang correctly points out in the comments, you should set CUDA_VISIBLE_DEVICES before importing Keras, otherwise your Tensorflow backend will occupy the memory of both GPUs. Doing so will most probably eliminate the discrepancy (it might also let you keep your high batch size without an OOM error).