AdaGrad.hpp

Go to the documentation of this file.

#ifndef ADAGRAD_HPP_
#define ADAGRAD_HPP_

#include <optimization/OptimizationFunction.hpp>

namespace mic {
namespace neural_nets {
namespace optimization {

template <typename eT=float>
class AdaGrad : public OptimizationFunction<eT> {
public:

    AdaGrad(size_t rows_, size_t cols_, eT eps_ = 1e-8) : eps(eps_) {
        G = MAKE_MATRIX_PTR(eT, rows_, cols_);
        // Reset G.
        G->zeros();

        // Allocate and reset delta.
        delta = MAKE_MATRIX_PTR(eT, rows_, cols_);
        delta->zeros();
    }

    mic::types::MatrixPtr<eT> calculateUpdate(mic::types::MatrixPtr<eT> x_, mic::types::MatrixPtr<eT> dx_, eT learning_rate_) {
        assert(x_->size() == dx_->size());
        assert(x_->size() == G->size());

        // Update G - add square of the gradients.
        for (size_t i=0; i<(size_t)x_->size(); i++)
                (*G)[i] += (*dx_)[i] * (*dx_)[i];

        // delta = alpha * dW.
        for (size_t i=0; i<(size_t)x_->size(); i++)
            (*delta)[i] = learning_rate_ * (*dx_)[i] / (std::sqrt((*G)[i] + eps));

        // Return the update.
        return delta;
    }

protected:
    eT eps;

    mic::types::MatrixPtr<eT> G;

    mic::types::MatrixPtr<eT> delta;
};

} //: optimization
} //: neural_nets
} //: mic

#endif /* ADAGRAD_HPP_ */