HebbianLinear.hpp

Go to the documentation of this file.

#ifndef SRC_MLNN_HEBBIANLINEAR_HPP_
#define SRC_MLNN_HEBBIANLINEAR_HPP_

#include <mlnn/layer/Layer.hpp>

namespace mic {
namespace mlnn {
namespace fully_connected {

template <typename eT=float>
class HebbianLinear : public mic::mlnn::Layer<eT> {
public:

    HebbianLinear(size_t inputs_, size_t outputs_, eT permanence_threshold_ = 0.5, eT proximal_threshold_ = 0.5, std::string name_ = "HebbianLinear") :
        HebbianLinear(inputs_, 1, 1, outputs_, 1, 1, permanence_threshold_, proximal_threshold_, name_)
    {

    }

    HebbianLinear(size_t input_height_, size_t input_width_, size_t input_depth_,
            size_t output_height_, size_t output_width_, size_t output_depth_,
            eT permanence_threshold_ = 0.5, eT proximal_threshold_ = 0.5,
            std::string name_ = "HebbianLinear") :
        Layer<eT>::Layer(input_height_, input_width_, input_depth_,
                output_height_, output_width_, output_depth_,
                LayerTypes::HebbianLinear, name_)
    {
        // Create the weights matrix.
        p.add ("W", Layer<eT>::outputSize(), Layer<eT>::inputSize());

        // Initialize weights of the W matrix.
        double range = sqrt(6.0 / double(Layer<eT>::outputSize() + Layer<eT>::inputSize()));
        Layer<eT>::p['W']->rand(-range, range);

        // Set hebbian learning as default optimization function.
        Layer<eT>::template setOptimization<mic::neural_nets::learning::HebbianRule<eT> > ();
    };


    virtual ~HebbianLinear() {};

    void forward(bool test_ = false) {
        // Get input matrices.
        mic::types::Matrix<eT> x = (*s['x']);
        mic::types::Matrix<eT> W = (*p['W']);
        // Get output pointer - so the results will be stored!
        mic::types::MatrixPtr<eT> y = s['y'];

        // Forward pass.
        (*y) = W * x;
        for (size_t i = 0; i < (size_t)s['x']->rows() * s['x']->cols(); i++) {
            // Sigmoid.
            //(*y)[i] = 1.0f / (1.0f +::exp(-(*y)[i]));
            // Threshold.
            (*y)[i] = ((*y)[i] > 0.8) ? 1.0f : 0.0f;
        }//: for
    }

    void backward() {
        throw std::logic_error("Backward propagation should not be used with layers using Hebbian learning!");
    }

    void update(eT alpha_, eT decay_  = 0.0f) {
        opt["W"]->update(p['W'], s['x'], s['y'], alpha_);
    }

    std::vector< std::shared_ptr <mic::types::Matrix<eT> > > & getActivations(size_t height_, size_t width_) {
        // Check if memory for the activations was allocated.
        if (neuron_activations.size() == 0) {
            for (size_t i=0; i < outputSize(); i++) {
                // Allocate memory for activation of every neuron.
                mic::types::MatrixPtr<eT> row = MAKE_MATRIX_PTR(eT, inputSize(), 1);
                neuron_activations.push_back(row);
            }//: for
        }//: if

        // Epsilon added for numerical stability.
        eT eps = 1e-10;

        mic::types::MatrixPtr<eT> W =  p["W"];
        // Iterate through "neurons" and generate "activation image" for each one.
        for (size_t i=0; i < outputSize(); i++) {
            // Get row.
            mic::types::MatrixPtr<eT> row = neuron_activations[i];
            // Copy data.
            (*row) = W->row(i);
            // Resize row.
            row->resize( height_, width_);
            // Calculate l2 norm.
            eT l2 = row->norm() + eps;
            // Normalize the inputs to <-0.5,0.5> and add 0.5f -> range <0.0, 1.0>.
            (*row) = row->unaryExpr ( [&] ( eT x ) { return ( x / l2 + 0.5f); } );
        }//: for

        // Return activations.
        return neuron_activations;
    }


    // Unhide the overloaded methods inherited from the template class Layer fields via "using" statement.
    using Layer<eT>::forward;
    using Layer<eT>::backward;

protected:
    // Unhide the fields inherited from the template class Layer via "using" statement.
    using Layer<eT>::s;
    using Layer<eT>::p;
    using Layer<eT>::m;
    using Layer<eT>::inputSize;
    using Layer<eT>::outputSize;
    using Layer<eT>::batch_size;
    using Layer<eT>::opt;

private:
    // Friend class - required for using boost serialization.
    template<typename tmp> friend class mic::mlnn::MultiLayerNeuralNetwork;
    //template<typename tmp> friend class mic::mlnn::HebbianNeuralNetwork;

    std::vector< std::shared_ptr <mic::types::Matrix<eT> > > neuron_activations;

    HebbianLinear<eT>() : Layer<eT> () { }

};


} /* namespace fully_connected */
} /* namespace mlnn */
} /* namespace mic */

#endif /* SRC_MLNN_HEBBIANLINEAR_HPP_ */