Keras kernel initializer with gabor filter

Question

I am trying create a custom initializer with Gabor kernels according to Keras documentation.

I am writing this example for an easy run and debug the code.

import random
from cv2.cv2 import CV_64F
import keras.backend as K
import numpy as np
from keras.layers import Conv2D
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from keras.utils import np_utils
from keras.layers.convolutional import MaxPooling2D
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.models import Sequential
import cv2
from theano import shared


images_size = 100
img_rows = 200
img_cols = 200
nb_channel = 3


def custom_gabor(shape, dtype=None):
    total_ker = []
    for i in xrange(shape[3]):
        kernels = []
        for j in xrange(shape[2]):
            kernels.append(
            cv2.getGaborKernel(ksize=(shape[0], shape[1]), sigma=1, 
            theta=1, lambd=0.5, gamma=0.3, psi=(3.14) * 0.5, 
            ktype=CV_64F))
        total_ker.append(kernels)
    np_tot = shared(np.array(total_ker))
    return K.variable(np_tot, dtype=dtype)


def build_model():
    model = Sequential()
    # Layer 1
    model.add(Conv2D(32, (3, 3), kernel_initializer=custom_gabor,
                                      input_shape=(nb_channel, img_rows, img_cols)))
    model.add(Activation('relu'))
    # Layer 2
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Conv2D(32, (3, 3), kernel_initializer=custom_gabor))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Layer 3
    model.add(Conv2D(32, (3, 3), kernel_initializer=custom_gabor))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(64))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(2))
    model.add(Activation('softmax'))

    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model


def make_dummy_data_set():
    img_matrix = []
    for i in xrange(images_size):
        img_matrix.append([random.random() for _ in xrange(img_rows*img_cols*nb_channel)])

    img_matrix = np.array(img_matrix)
    label = np.array([random.randint(0, 1) for _ in xrange(images_size)])

    data, label = shuffle(img_matrix, label, random_state=7)  # random_state=2
    X_train, X_test, y_train, y_test = train_test_split(data, label, test_size=0.2, random_state=7)

    # reshape the data
    X_train = X_train.reshape(X_train.shape[0], nb_channel, img_rows, img_cols)
    X_test = X_test.reshape(X_test.shape[0], nb_channel, img_rows, img_cols)

    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')

    # convert class vectore to binary class matrices
    y_train = np_utils.to_categorical(y_train, 2)
    y_test = np_utils.to_categorical(y_test, 2)

    return X_train, X_test, y_train, y_test


def train_model(model, X_train, X_test, y_train, y_test):
    model.fit(X_train,
              y_train,
              batch_size=32,
              epochs=5,
              verbose=1,
              validation_data=(X_test, y_test))


if __name__ == "__main__":
    model = build_model()
    X_train, X_test, y_train, y_test = make_dummy_data_set()
    train_model(model, X_train, X_test, y_train, y_test)

when I run it I get the error "Input dimension mis-match":

Using Theano backend.
Train on 80 samples, validate on 20 samples
Epoch 1/5
Traceback (most recent call last):
File "/home/naor/Desktop/workspace/reflux_analyze/core/tests/test.py", 
line 104, in <module>
train_model(model, X_train, X_test, y_train, y_test)
File "/home/naor/Desktop/workspace/reflux_analyze/core/tests/test.py", line 98, in train_model
validation_data=(X_test, y_test))
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/keras/models.py", line 867, in fit
initial_epoch=initial_epoch)
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/keras/engine/training.py", line 1598, in fit
validation_steps=validation_steps)
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/keras/engine/training.py", line 1183, in _fit_loop
outs = f(ins_batch)
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 1222, in __call__
return self.function(*inputs)
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 898, in __call__
storage_map=getattr(self.fn, 'storage_map', None))
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/theano/gof/link.py", line 325, in raise_with_op
reraise(exc_type, exc_value, exc_trace)
File "/home/naor/Desktop/workspace/reflux_analyze/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 884, in __call__
self.fn() if output_subset is None else\
ValueError: Input dimension mis-match. (input[0].shape[1] = 3, input[1].shape[1] = 32)
Apply node that caused the error: Elemwise{Add}[(0, 0)](CorrMM{valid, 
(1, 1), (1, 1)}.0, InplaceDimShuffle{x,0,x,x}.0)
Toposort index: 73
Inputs types: [TensorType(float32, 4D), TensorType(float32, (True, 
False, True, True))]
Inputs shapes: [(32, 3, 169, 198), (1, 32, 1, 1)]
Inputs strides: [(401544, 133848, 792, 4), (128, 4, 4, 4)]
Inputs values: ['not shown', 'not shown']
Outputs clients: [[Elemwise{Composite{(i0 * (i1 + Abs(i1)))}}
(TensorConstant{(1, 1, 1, 1) of 0.5}, Elemwise{Add}[(0, 0)].0), 
Elemwise{Composite{((i0 * i1) + (i0 * i1 * sgn(i2)))}}[(0, 1)]
(TensorConstant{(1, 1, 1, 1) of 0.5}, MaxPoolGrad{ignore_border=True, 
mode='max', ndim=2}.0, Elemwise{Add}[(0, 0)].0)]]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.

Process finished with exit code 1

I know this kernel will change by CNN learning phase but i want to start from thows gabor kernel (only the real part) And my data is a pictures and not a random... :)

thanks.

Answer 1

From the input shapes in your error message, we can identify that (32,3,169,198) is your input images.

• They are 32 images (batch_size), with 3 channels, and sides 169,198.

Then I assume the other shape (1,32,1,1) is your filter shape.

The output shape of your filters must be 32. Ok, there is a 32 there, but I'm not sure it's in the right place. (As soon as I get the print of the shape I asked in the other comment, I can update the order of these dimensions -- but for now, I'm running important code, I can't change my settings)

The other dimensions are all wrong, though. Your filters should be shaped as:

• 32, relating to the 32 cells (output channels) in your convolutional layer
• 3 (nb_channel), relating to the input channels coming into the layer
• 3, relating to kernel size 1
• 3, relating to kernel size 2

Answer 2

def custom_gabor(shape, dtype=None):
    total_ker = []
    for i in xrange(shape[0]):
        kernels = []
        for j in xrange(shape[1]):
        # gk = gabor_kernel(frequency=0.2, bandwidth=0.1)
        tmp_filter = cv2.getGaborKernel(ksize=(shape[3], shape[2]), sigma=1, theta=1, lambd=0.5, gamma=0.3, psi=(3.14) * 0.5,
                           ktype=CV_64F)
            filter = []
            for row in tmp_filter:
                filter.append(np.delete(row, -1))
            kernels.append(filter)
                # gk.real
        total_ker.append(kernels)
    np_tot = shared(np.array(total_ker))
    return K.variable(np_tot, dtype=dtype)