Class implementing a linear, fully connected layer. More...

#include <Linear.hpp>

Inheritance diagram for mic::mlnn::fully_connected::Linear< eT >:

Collaboration diagram for mic::mlnn::fully_connected::Linear< eT >:

Public Member Functions
	Linear (size_t inputs_, size_t outputs_, std::string name_="Linear")

	Linear (size_t input_height_, size_t input_width_, size_t input_depth_, size_t output_height_, size_t output_width_, size_t output_depth_, std::string name_="Linear")

virtual	~Linear ()

void	forward (bool test_=false)

void	backward ()

void	resetGrads ()

void	update (eT alpha_, eT decay_=0.0f)

std::vector < mic::types::MatrixPtr< eT > > &	getWeightActivations ()

std::vector < mic::types::MatrixPtr< eT > > &	getWeightGradientActivations ()

std::vector < mic::types::MatrixPtr< eT > > &	getInverseWeightActivations ()

std::vector < mic::types::MatrixPtr< eT > > &	getInverseOutputActivations ()

eT	calculateMeanReconstructionError ()

Public Member Functions inherited from mic::mlnn::Layer< eT >
	Layer (size_t input_height_, size_t input_width_, size_t input_depth_, size_t output_height_, size_t output_width_, size_t output_depth_, LayerTypes layer_type_, std::string name_="layer")

virtual	~Layer ()

mic::types::MatrixPtr< eT >	forward (mic::types::MatrixPtr< eT > x_, bool test=false)

mic::types::MatrixPtr< eT >	backward (mic::types::MatrixPtr< eT > dy_)

virtual void	resizeBatch (size_t batch_size_)

template<typename loss >
mic::types::MatrixPtr< eT >	calculateNumericalGradient (mic::types::MatrixPtr< eT > x_, mic::types::MatrixPtr< eT > target_y_, mic::types::MatrixPtr< eT > param_, loss loss_, eT delta_)

size_t	inputSize ()
	Returns size (length) of inputs. More...

size_t	outputSize ()
	Returns size (length) of outputs. More...

size_t	batchSize ()
	Returns size (length) of (mini)batch. More...

const std::string	name () const
	Returns name of the layer. More...

mic::types::MatrixPtr< eT >	getParam (std::string name_)

mic::types::MatrixPtr< eT >	getState (std::string name_)

mic::types::MatrixPtr< eT >	getGradient (std::string name_)

void	setState (std::string name_, mic::types::MatrixPtr< eT > mat_ptr_)

template<typename omT >
void	setOptimization ()

const std::string	type () const

virtual std::string	streamLayerParameters ()

mic::types::MatrixPtr< eT >	lazyReturnSampleFromBatch (mic::types::MatrixPtr< eT > batch_ptr_, mic::types::MatrixArray< eT > &array_, std::string id_, size_t sample_number_, size_t sample_size_)

mic::types::MatrixPtr< eT >	lazyReturnInputSample (mic::types::MatrixPtr< eT > batch_ptr_, size_t sample_number_)

mic::types::MatrixPtr< eT >	lazyReturnOutputSample (mic::types::MatrixPtr< eT > batch_ptr_, size_t sample_number_)

mic::types::MatrixPtr< eT >	lazyReturnChannelFromSample (mic::types::MatrixPtr< eT > sample_ptr_, mic::types::MatrixArray< eT > &array_, std::string id_, size_t sample_number_, size_t channel_number_, size_t height_, size_t width_)

mic::types::MatrixPtr< eT >	lazyReturnInputChannel (mic::types::MatrixPtr< eT > sample_ptr_, size_t sample_number_, size_t channel_number_)

mic::types::MatrixPtr< eT >	lazyReturnOutputChannel (mic::types::MatrixPtr< eT > sample_ptr_, size_t sample_number_, size_t channel_number_)

void	lazyAllocateMatrixVector (std::vector< std::shared_ptr< mic::types::Matrix< eT > > > &vector_, size_t vector_size_, size_t matrix_height_, size_t matrix_width_)

virtual std::vector < std::shared_ptr < mic::types::Matrix< eT > > > &	getInputActivations ()

virtual std::vector < std::shared_ptr < mic::types::Matrix< eT > > > &	getInputGradientActivations ()

virtual std::vector < std::shared_ptr < mic::types::Matrix< eT > > > &	getOutputActivations ()

virtual std::vector < std::shared_ptr < mic::types::Matrix< eT > > > &	getOutputGradientActivations ()

Private Member Functions
	Linear ()

Private Attributes
std::vector < mic::types::MatrixPtr< eT > >	w_activations
	Vector containing activations of weights/filters. More...

std::vector < mic::types::MatrixPtr< eT > >	dw_activations
	Vector containing activations of gradients of weights (dW). More...

std::vector < mic::types::MatrixPtr< eT > >	inverse_w_activations
	Vector containing "inverse activations" of each neuron weights(W^T). More...

std::vector < mic::types::MatrixPtr< eT > >	inverse_y_activations
	Vector containing activations of neurons (y*W^T). More...

Friends
template<typename tmp >
class	mic::mlnn::MultiLayerNeuralNetwork

template<typename tmp >
class	mic::mlnn::fully_connected::SparseLinear

Additional Inherited Members
Protected Member Functions inherited from mic::mlnn::Layer< eT >
	Layer ()

Protected Attributes inherited from mic::mlnn::Layer< eT >
size_t	input_height
	Height of the input (e.g. 28 for MNIST). More...

size_t	input_width
	Width of the input (e.g. 28 for MNIST). More...

size_t	input_depth
	Number of channels of the input (e.g. 3 for RGB images). More...

size_t	output_height
	Number of receptive fields in a single channel - vertical direction. More...

size_t	output_width
	Number of receptive fields in a single channel - horizontal direction. More...

size_t	output_depth
	Number of filters = number of output channels. More...

size_t	batch_size
	Size (length) of (mini)batch. More...

LayerTypes	layer_type
	Type of the layer. More...

std::string	layer_name
	Name (identifier of the type) of the layer. More...

mic::types::MatrixArray< eT >	s
	States - contains input [x] and output [y] matrices. More...

mic::types::MatrixArray< eT >	g
	Gradients - contains input [x] and output [y] matrices. More...

mic::types::MatrixArray< eT >	p
	Parameters - parameters of the layer, to be used by the derived classes. More...

mic::types::MatrixArray< eT >	m
	Memory - a list of temporal parameters, to be used by the derived classes. More...

mic::neural_nets::optimization::OptimizationArray < eT >	opt
	Array of optimization functions. More...

std::vector< std::shared_ptr < mic::types::Matrix< eT > > >	x_activations
	Vector containing activations of input neurons - used in visualization. More...

std::vector< std::shared_ptr < mic::types::Matrix< eT > > >	dx_activations
	Vector containing activations of gradients of inputs (dx) - used in visualization. More...

std::vector< std::shared_ptr < mic::types::Matrix< eT > > >	y_activations
	Vector containing activations of output neurons - used in visualization. More...

std::vector< std::shared_ptr < mic::types::Matrix< eT > > >	dy_activations
	Vector containing activations of gradients of outputs (dy) - used in visualization. More...

Detailed Description

template<typename eT = float>
class mic::mlnn::fully_connected::Linear< eT >

Class implementing a linear, fully connected layer.

Author: tkornuta

Template Parameters

eT	Template parameter denoting precision of variables (float for calculations/double for testing).

Definition at line 42 of file Linear.hpp.

Constructor & Destructor Documentation

template<typename eT = float>

mic::mlnn::fully_connected::Linear< eT >::Linear	(	size_t	inputs_,
		size_t	outputs_,
		std::string	name_ = `"Linear< eT >"`
	)

inline

Creates a linear (i.e fully connected) layer - reduced number of parameters.

Parameters

inputs_	Length of the input vector.
outputs_	Length of the output vector.
name_	Name of the layer.

Definition at line 51 of file Linear.hpp.

template<typename eT = float>

mic::mlnn::fully_connected::Linear< eT >::Linear	(	size_t	input_height_,
		size_t	input_width_,
		size_t	input_depth_,
		size_t	output_height_,
		size_t	output_width_,
		size_t	output_depth_,
		std::string	name_ = `"Linear< eT >"`
	)

inline

Creates a linear (i.e fully connected) layer.

Parameters

input_height_	Height of the input sample.
input_width_	Width of the input sample.
input_depth_	Depth of the input sample.
output_height_	Width of the output sample.
output_width_	Height of the output sample.
output_depth_	Depth of the output sample.
name_	Name of the layer.

Definition at line 68 of file Linear.hpp.

template<typename eT = float>

virtual mic::mlnn::fully_connected::Linear< eT >::~Linear ( )

inlinevirtual

Virtual destructor - empty.

Definition at line 99 of file Linear.hpp.

template<typename eT = float>

mic::mlnn::fully_connected::Linear< eT >::Linear ( )

inlineprivate

Private constructor, used only during the serialization.

Definition at line 367 of file Linear.hpp.

Member Function Documentation

template<typename eT = float>

void mic::mlnn::fully_connected::Linear< eT >::backward ( )

inlinevirtual

Backward pass.

Implements mic::mlnn::Layer< eT >.

Reimplemented in mic::mlnn::fully_connected::SparseLinear< eT >.

Definition at line 126 of file Linear.hpp.

Referenced by mic::neural_nets::unit_tests::TEST().

template<typename eT = float>

eT mic::mlnn::fully_connected::Linear< eT >::calculateMeanReconstructionError ( )

inline

Calculates mean reconstruction error (mean of all RMS errors between input and reconstructed samples).

Definition at line 289 of file Linear.hpp.

template<typename eT = float>

void mic::mlnn::fully_connected::Linear< eT >::forward ( bool test_ = false )

inlinevirtual

Forward pass.

Parameters

test_ It ise set to true in test mode (network verification).

Implements mic::mlnn::Layer< eT >.

Definition at line 105 of file Linear.hpp.

Referenced by mic::neural_nets::unit_tests::TEST().

template<typename eT = float>

std::vector< mic::types::MatrixPtr<eT> >& mic::mlnn::fully_connected::Linear< eT >::getInverseOutputActivations ( )

inline

Returns inverse activations of output neurons (y*W^T) - reconstruction of the input.

Definition at line 249 of file Linear.hpp.

Referenced by mic::mlnn::fully_connected::Linear< float >::calculateMeanReconstructionError().

template<typename eT = float>

std::vector< mic::types::MatrixPtr<eT> >& mic::mlnn::fully_connected::Linear< eT >::getInverseWeightActivations ( )

inline

Returns "inverse activations" of each neuron weights (W^T) - shows what activates given neuron.

Definition at line 219 of file Linear.hpp.

template<typename eT = float>

std::vector< mic::types::MatrixPtr<eT> >& mic::mlnn::fully_connected::Linear< eT >::getWeightActivations ( )

inline

Returns activations of weights.

Definition at line 173 of file Linear.hpp.

template<typename eT = float>

std::vector< mic::types::MatrixPtr<eT> >& mic::mlnn::fully_connected::Linear< eT >::getWeightGradientActivations ( )

inline

Returns activations of weight gradients (dx).

Definition at line 196 of file Linear.hpp.

template<typename eT = float>

void mic::mlnn::fully_connected::Linear< eT >::resetGrads ( )

inlinevirtual

Resets the gradients for W and b.

Reimplemented from mic::mlnn::Layer< eT >.

Definition at line 148 of file Linear.hpp.

template<typename eT = float>

void mic::mlnn::fully_connected::Linear< eT >::update	(	eT	alpha_,
		eT	decay_ = `0.0f`
	)

inlinevirtual

Applies the gradient update, using the selected optimization method.

Parameters

alpha_	Learning rate - passed to the optimization functions of all layers.
decay_	Weight decay rate (determining that the "unused/unupdated" weights will decay to 0) (DEFAULT=0.0 - no decay).

Implements mic::mlnn::Layer< eT >.

Reimplemented in mic::mlnn::fully_connected::SparseLinear< eT >.

Definition at line 159 of file Linear.hpp.