encoder/decoder ripping from autoencoder in encog

Question

I have created and learned autoencoder in Encog and I try to rip it into parts: encoder and decoder part. Unfortunately I cannot get it and I keep getting strange improper data (comparing result from applying once net to data and twice data -> enc -> dec).

I have tried to make it with simply GetWeight and SetWeight but there result is incorrect. The solution found in encog documentation - initialization flat network is for me not clear (I cannot get it working).

        public static BasicNetwork getEncoder(BasicNetwork net)
        {
            var enc = new BasicNetwork();
            enc.AddLayer(new BasicLayer(null, true, net.GetLayerNeuronCount(0)));
            enc.AddLayer(new BasicLayer(new ActivationSigmoid(), true, net.GetLayerNeuronCount(1)));
            enc.AddLayer(new BasicLayer(new ActivationSigmoid(), false, net.GetLayerNeuronCount(2)));
            enc.Structure.FinalizeStructure ();

            var weights1 = net.Structure.Flat.Weights;
            var weights2 = enc.Structure.Flat.Weights;
            var idx1 = net.Structure.Flat.WeightIndex;
            var idx2 = enc.Structure.Flat.WeightIndex;

            for(var i = 0; i < 1; i++)
            {
                int n = net.GetLayerNeuronCount(i);
                int m = net.GetLayerNeuronCount(i + 1);

                Console.WriteLine("Decoder: {0} - {1}", n, m);

                for(var j = 0; j < n; j++)
                {
                    for(var k = 0; k < m; k++)
                    {
                        weights1 [idx1[i] + j * m + k] = weights2 [idx2[i] + j * m * k];
                    }
                }
            }


            return enc;
        }

Full old-like (set/get weight) code of AutoEncoder:

using System;
using Encog.Engine.Network.Activation;
using Encog.ML.Data;
using Encog.ML.Data.Basic;
using Encog.ML.Train;
using Encog.Neural.Networks;
using Encog.Neural.Networks.Layers;
using Encog.Neural.Networks.Training.Propagation.Resilient;

namespace engine
{
    public class AutoEncoder
    {
        private int k = 0;
        public IMLDataSet trainingSet
        {
            get;
            set;
        }

        public AutoEncoder(int k)
        {
            this.k = k;
        }

        public static BasicNetwork getDecoder(BasicNetwork net)
        {
            var dec = new BasicNetwork();
            dec.AddLayer(new BasicLayer(null, true, net.GetLayerNeuronCount(1)));
            dec.AddLayer(new BasicLayer(new ActivationSigmoid(), true, net.GetLayerNeuronCount(2)));

            dec.Structure.FinalizeStructure();

            for(var i = 1; i < 2; i++)
            {
                int n = net.GetLayerNeuronCount(i);
                int m = net.GetLayerNeuronCount(i + 1);

                Console.WriteLine("Decoder: {0} - {1}", n, m);

                for(var j = 0; j < n; j++)
                {
                    for(var k = 0; k < m; k++)
                    {
                        dec.SetWeight(i - 1, j, k, net.GetWeight(i, j, k));
                    }
                }
            }

            return dec;
        }

        public static BasicNetwork getEncoder(BasicNetwork net)
        {
            var enc = new BasicNetwork();
            enc.AddLayer(new BasicLayer(null, true, net.GetLayerNeuronCount(0)));
            enc.AddLayer(new BasicLayer(new ActivationSigmoid(), true, net.GetLayerNeuronCount(1)));

            enc.Structure.FinalizeStructure();

            for(var i = 0; i < 1; i++)
            {
                int n = net.GetLayerNeuronCount(i);
                int m = net.GetLayerNeuronCount(i + 1);

                Console.WriteLine("Encoder: {0} - {1}", n, m);

                for(var j = 0; j < n; j++)
                {
                    for(var k = 0; k < m; k++)
                    {
                        enc.SetWeight(i, j, k, net.GetWeight(i, j, k));
                    }
                }
            }

            return enc;
        }

        public BasicNetwork learn(double[][] data,
            double eps = 1e-6,
            long trainMaxIter = 10000)
        {
            int n = data.Length;
            int m = data[0].Length;
            double[][] output = new double[n][];
            for(var i = 0; i < n; i++)
            {
                output[i] = new double[m];
                data[i].CopyTo(output[i], 0);
            }

            var network = new BasicNetwork();
            network.AddLayer(new BasicLayer(null, true, m));
            network.AddLayer(new BasicLayer(new ActivationSigmoid(), true, k));
            network.AddLayer(new BasicLayer(new ActivationSigmoid(), true, m));
            network.Structure.FinalizeStructure();
            network.Reset();

            trainingSet = new BasicMLDataSet(data, output);
            IMLTrain train = new ResilientPropagation(network, trainingSet);

            int epoch = 1;
            do
            {
                train.Iteration();
                Console.WriteLine(@"Epoch #" + epoch + @" Error:" + train.Error);
                epoch++;
            } while(train.Error > eps && epoch < trainMaxIter);

            train.FinishTraining();

            return network;
        }
    }
}

How can I correctly rip only two first layers from autoencoder for encoder and two last layers from one for decoder?

Answer 1

If you need direct access to the weights, the best method is to use BasicNetwork.GetWeight(). Here is an example that shows how to use GetWeight to obtain all of the weights in the neural network. It is from a unit test, to prove that GetWeight does work, it calculates the output of a simple neural network using BasicNetwork.Compute and also manually just by summing the weighted inputs and applying the TanH. Both result in the same output.

More info here too, if you want to access the weight array directly: http://www.heatonresearch.com/wiki/Weight

        var network = new BasicNetwork();
        network.AddLayer(new BasicLayer(null, true, 2));
        network.AddLayer(new BasicLayer(new ActivationTANH(), true, 2));
        network.AddLayer(new BasicLayer(new ActivationTANH(), false, 1));
        network.Structure.FinalizeStructure();
        network.Reset(100);

        BasicMLData input = new BasicMLData(2);
        input[0] = 0.1;
        input[1] = 0.2;

        Console.WriteLine("Using network: " + network.Compute(input));

        // now manually
        double sum1 = (input[0]*network.GetWeight(0, 0, 0))
                      + (input[1]*network.GetWeight(0, 1, 0))
                      + (1.0*network.GetWeight(0,2,0));

        double sum2 = (input[0]*network.GetWeight(0, 0, 1))
                      + (input[1]*network.GetWeight(0, 1, 1))
                      + (1.0*network.GetWeight(0,2,1));

        double hidden1 = Math.Tanh(sum1);
        double hidden2 = Math.Tanh(sum2);

        double sum3 = (hidden1 * network.GetWeight(1, 0, 0))
                      + (hidden2 * network.GetWeight(1, 1, 0))
                      + (1.0 * network.GetWeight(1, 2, 0));

        double output = Math.Tanh(sum3);

        Console.WriteLine("Using manual: " + network.Compute(input));