AdaDelta.hpp

Go to the documentation of this file.

#ifndef ADADELTA_HPP_
#define ADADELTA_HPP_

#include <optimization/OptimizationFunction.hpp>

namespace mic {
namespace neural_nets {
namespace optimization {

template <typename eT=float>
class AdaDelta : public OptimizationFunction<eT> {
public:

    AdaDelta(size_t rows_, size_t cols_, eT decay_ = 0.9, eT eps_ = 1e-8) : decay(decay_), eps(eps_) {
        EG = MAKE_MATRIX_PTR(eT, rows_, cols_);
        EG->zeros();

        ED = MAKE_MATRIX_PTR(eT, rows_, cols_);
        ED->zeros();

        // Allocate and reset delta.
        delta = MAKE_MATRIX_PTR(eT, rows_, cols_);
        delta->zeros();
    }

    mic::types::MatrixPtr<eT> calculateUpdate(mic::types::MatrixPtr<eT> x_, mic::types::MatrixPtr<eT> dx_, eT learning_rate_) {
        assert(x_->size() == dx_->size());
        assert(x_->size() == EG->size());

/*      for (size_t i=0; i<x_->size(); i++) {
            std::cout << "(*x)["<< i <<"] = " << (*x_)[i] <<std::endl;
        }
        for (size_t i=0; i<x_->size(); i++) {
            std::cout << "(*dx_)["<< i <<"] = " << (*dx_)[i] <<std::endl;
        }*/

        // Update decaying sum of squares of gradients - up to time t.
        for (size_t i=0; i<(size_t)x_->size(); i++) {
            (*EG)[i] = decay *(*EG)[i] + (1.0 - decay) * (*dx_)[i] * (*dx_)[i];
//          std::cout << "(*EG)["<< i <<"] = " << (*EG)[i] <<std::endl;
            assert(std::isfinite((*EG)[i]));
        }

        // Update decaying sum of squares of updates - up to time t-1.
        for (size_t i=0; i<(size_t)x_->size(); i++)
            (*ED)[i] = decay *(*ED)[i] + (1 - decay) * (*delta)[i] * (*delta)[i];

        // Calculate updates - and store as previous (already) = - RMS(ED)/(RMS(G) * dx
        for (size_t i=0; i<(size_t)x_->size(); i++){
//          (*prev_d)[i] = (0.1 / std::sqrt((*EG)[i] + eps)) * (*dx_)[i];
            (*delta)[i] = (std::sqrt((*ED)[i] + eps) / std::sqrt((*EG)[i] + eps)) * (*dx_)[i];
//          std::cout << "(*prev_d)["<< i <<"] = " << (*prev_d)[i] <<std::endl;
            assert(std::isfinite((*delta)[i]));
        }

        // Return the update.
        return delta;
    }

protected:
    eT decay;

    eT eps;

    mic::types::MatrixPtr<eT> EG;

    mic::types::MatrixPtr<eT> ED;

    mic::types::MatrixPtr<eT> delta;
};


} //: optimization
} //: neural_nets
} //: mic

#endif /* ADADELTA_HPP_ */