SparseLinear.hpp

Go to the documentation of this file.

#ifndef SPARSELINEAR_H_
#define SPARSELINEAR_H_

#include <mlnn/fully_connected/Linear.hpp>

namespace mic {
namespace mlnn {
namespace fully_connected {

template <typename eT=float>
class SparseLinear : public mic::mlnn::fully_connected::Linear<eT> {
public:
    SparseLinear<eT>(size_t inputs_, size_t outputs_, std::string name_ = "SparseLinear") :
        Linear<eT>(inputs_, outputs_, name_) {

        // Change type to SparseLinear.
        Layer<eT>::layer_type = LayerTypes::SparseLinear;

        // Prepare matrices in the "temporal memory".
        // For current sparsity vector.
        m.add ("ro", outputSize(), 1 );
        // For penalty.
        m.add ("penalty", outputSize(), 1 );

        // Set desired sparsity and penalty term.
        desired_ro = 0.1; // 10 %
        beta = 0.5;
    };


    virtual ~SparseLinear() {};

    void backward() {
        eT eps = 1e-10;
        // Calculate the current "activation sparsity".
        mic::types::MatrixPtr<eT> ro = m["ro"];
        (*ro) = ((*s['y']).rowwise().sum()/batch_size);

        // Calculate the sparsity penalty - for every output neuron.
        mic::types::MatrixPtr<eT> penalty = m["penalty"];
        for (size_t i=0; i<outputSize(); i++)
            (*penalty)[i] = beta*(-desired_ro/((*ro)[i] + eps) + (1-desired_ro)/(1-(*ro)[i] + eps));


        // Calculate derivatives of W,b and x.
        (*g['W']) = (*g['y']) * ((*s['x']).transpose());
        (*g['b']) = (*g['y']).rowwise().mean();
        (*g['x']) = (*p['W']).transpose() * (*g['y']);
    }

    void update(eT alpha_, eT decay_  = 0.0f) {
        //std::cout << "p['W'] = \n" << (*p['W']) << std::endl;
        //std::cout << "g['W'] = \n" << (*g['W']) << std::endl;

        // Apply selected learning rule to W.
        opt["W"]->update(p['W'], g['W'], alpha_, decay_);

        // Apply sparsity learning rule to b, incorporating the KL-divergence term.
        mic::types::MatrixPtr<eT> penalty = m["penalty"];
        // (*p['b']) -=  alpha_ * beta * (*penalty);
        opt["b"]->update(p['b'], g['b'], alpha_, 0.0);

        //std::cout << "p['W'] after update= \n" << (*p['W']) << std::endl;
    }

    // Unhide the overloaded methods inherited from the template class Layer fields via "using" statement.
    using Layer<eT>::forward;
    using Layer<eT>::backward;

protected:
    // Unhide the fields inherited from the template class Layer via "using" statement.
    using Layer<eT>::g;
    using Layer<eT>::s;
    using Layer<eT>::p;
    using Layer<eT>::m;
    using Layer<eT>::inputSize;
    using Layer<eT>::outputSize;
    using Layer<eT>::batch_size;
    using Layer<eT>::opt;

private:
    // Friend class - required for using boost serialization.
    template<typename tmp> friend class mic::mlnn::MultiLayerNeuralNetwork;

    SparseLinear<eT>() : mic::mlnn::fully_connected::Linear<eT> () { }

    eT desired_ro;

    eT beta;
};


} /* namespace fully_connected */
} /* namespace mlnn */
} /* namespace mic */

#endif /* SPARSELINEAR_H_ */