RMSProp.hpp

Go to the documentation of this file.

#ifndef RMSPROP_HPP_
#define RMSPROP_HPP_

#include <optimization/OptimizationFunction.hpp>

namespace mic {
namespace neural_nets {
namespace optimization {

template <typename eT=float>
class RMSProp : public OptimizationFunction<eT> {
public:

    RMSProp(size_t rows_, size_t cols_, eT decay_ = 0.9, eT eps_ = 1e-8) : decay(decay_), eps(eps_) {
        EG = MAKE_MATRIX_PTR(eT, rows_, cols_);
        delta = MAKE_MATRIX_PTR(eT, rows_, cols_);
        // Reset EG and delta.
        EG->zeros();
        delta->zeros();
    }

    mic::types::MatrixPtr<eT> calculateUpdate(mic::types::MatrixPtr<eT> x_, mic::types::MatrixPtr<eT> dx_, eT learning_rate_) {
        assert(x_->size() == dx_->size());
        assert(x_->size() == EG->size());

        // Update decaying sum of squares of gradients - up to time t.
        for (size_t i=0; i<(size_t)x_->size(); i++) {
            (*EG)[i] = decay *(*EG)[i] + (1.0 - decay) * (*dx_)[i] * (*dx_)[i];
            assert(std::isfinite((*EG)[i]));
        }

        // Calculate updates - and store as previous (already) = - RMS(ED)/(RMS(G) * dx
        for (size_t i=0; i<(size_t)x_->size(); i++){
            (*delta)[i] = (learning_rate_ / std::sqrt((*EG)[i] + eps)) * (*dx_)[i];
            assert(std::isfinite((*delta)[i]));
        }

        // Return the update.
        return delta;
    }

protected:
    eT decay;

    eT eps;

    mic::types::MatrixPtr<eT> EG;

    mic::types::MatrixPtr<eT> delta;
};

} //: optimization
} //: neural_nets
} //: mic

#endif /* RMSPROP_HPP_ */