Adam.hpp

Go to the documentation of this file.

#ifndef ADAM_HPP_
#define ADAM_HPP_

#include <optimization/OptimizationFunction.hpp>

namespace mic {
namespace neural_nets {
namespace optimization {

template <typename eT=float>
class Adam : public OptimizationFunction<eT> {
public:

    Adam(size_t rows_, size_t cols_, eT beta1_ = 0.9, eT beta2_ = 0.999, eT eps_ = 1e-8)
            : beta1(beta1_), beta2(beta2_), eps(eps_)
{
        m = MAKE_MATRIX_PTR(eT, rows_, cols_);
        m->zeros();

        v = MAKE_MATRIX_PTR(eT, rows_, cols_);
        v->zeros();

        // Allocate and reset delta.
        delta = MAKE_MATRIX_PTR(eT, rows_, cols_);
        delta->zeros();

        beta1_powt = beta1;
        beta2_powt = beta2;
    }

    mic::types::MatrixPtr<eT> calculateUpdate(mic::types::MatrixPtr<eT> x_, mic::types::MatrixPtr<eT> dx_, eT learning_rate_  = 0.001) {
        assert(x_->size() == dx_->size());
        assert(x_->size() == m->size());

        // Update the decaying average of past gradients.
        for (size_t i=0; i<(size_t)x_->size(); i++)
            (*m)[i] = beta1 * (*m)[i] + (1-beta1) * (*dx_)[i];

        // Update the decaying average of past squared gradients.
        for (size_t i=0; i<(size_t)x_->size(); i++)
            (*v)[i] = beta2 * (*v)[i] + (1-beta2) * (*dx_)[i] * (*dx_)[i];

        // Calculate the update.
        for (size_t i=0; i<(size_t)x_->size(); i++)
            (*delta)[i] = learning_rate_ / (sqrt( (*v)[i] / (1 - beta2_powt)) + eps) * (*m)[i] / (1 - beta1_powt);

        // Update "powered" factors.
        beta1_powt *= beta1;
        beta2_powt *= beta2;

        // Return the update.
        return delta;
    }

protected:
    mic::types::MatrixPtr<eT> m;

    mic::types::MatrixPtr<eT> v;

    mic::types::MatrixPtr<eT> delta;

    eT beta1;

    eT beta2;

    eT eps;

    eT beta1_powt;

    eT beta2_powt;

};

} //: optimization
} //: neural_nets
} //: mic

#endif /* ADAM_HPP_ */