Why doesn't this code work? - Backpropagation algorithm

49 views Asked by At

I have been trying to write a working implementation for the backpropagation for around four months now and I just can't get it to work. Here is my code:

import numpy as np 

class neural_network:
    def __init__(self, layers, alpha = 0.01):
        self.alpha = alpha
        self.activate = (lambda x: 1.0 / (1.0 + np.exp(-x)))
        self.rate = (lambda x: x * (1.0 - x))
        self.W = list()
        for i in np.arange(0, len(layers) - 2):
            self.W.append(np.random.randn(layers[i] + 1, layers[i + 1] + 1))
        i += 1 
        self.W.append(np.random.randn(layers[i] + 1, layers[i + 1]))

    def forward(self, I):
        I = np.atleast_2d(I)
        I = np.c_[I, np.ones(I.shape[0])]
        z = np.matmul(I, self.W[0])
        a = self.activate(z)
        self.A = [a]
        for i in np.arange(0, len(self.W)):
            z = np.matmul(self.A[-1], self.W[i])
            a = self.activate(z)
            self.A.append(a)
        return a

    def error(self, O, Y):
        return (1 / 2) * np.mean((O - Y) ** 2)

    def backward(self, I, Y):
        O = self.forward(I)
        error = O - Y
        D = [self.rate(self.A[-1]) * error]
        for i in np.arange(len(self.W) - 1, 0, -1):
            d = np.matmul(D[-1], self.W[i].T) * self.rate(self.A[i])
            D.append(d)
        D = D[::-1]
        for i in np.arange(0, len(self.W)):
            self.W[i] -= np.matmul(self.A[i].T, D[i]) * self.alpha
            
if __name__ == '__main__':
    nn = neural_network([2, 2, 1])
    I = np.array([[0, 1], [1, 0], [1, 1], [0, 0]])
    Y = np.array([[1], [1], [1], [0]])
    O = nn.forward(I)
    print(nn.error(O, Y))
    for i in range(10000):
        nn.backward(I, Y)
    O = nn.forward(I)
    print(O)
    print(nn.error(O, Y))
    exit(0)

Here is what I'm using as reference:

http://neuralnetworksanddeeplearning.com/chap2.html

Can someone please help me find the mistake? I really don't seem to be able to find it by myself.

Thank you beforehand.

Write an implementation for the backpropagation algorithm. I was expecting the algorithm to be able to train the network but it doesn't.

0

There are 0 answers