parametric_relu.hpp
Go to the documentation of this file.
1 
15 #ifndef MLPACK_METHODS_ANN_LAYER_PReLU_HPP
16 #define MLPACK_METHODS_ANN_LAYER_PReLU_HPP
17 
18 #include <mlpack/prereqs.hpp>
19 
20 namespace mlpack {
21 namespace ann {
22 
41 template <
42  typename InputDataType = arma::mat,
43  typename OutputDataType = arma::mat
44 >
45 class PReLU
46 {
47  public:
56  PReLU(const double userAlpha = 0.03);
57 
58  /*
59  * Reset the layer parameter.
60  */
61  void Reset();
62 
70  template<typename InputType, typename OutputType>
71  void Forward(const InputType&& input, OutputType&& output);
72 
82  template<typename DataType>
83  void Backward(const DataType&& input, DataType&& gy, DataType&& g);
84 
92  template<typename eT>
93  void Gradient(const arma::Mat<eT>&& input,
94  arma::Mat<eT>&& error,
95  arma::Mat<eT>&& gradient);
96 
98  OutputDataType const& Parameters() const { return alpha; }
100  OutputDataType& Parameters() { return alpha; }
101 
103  OutputDataType const& OutputParameter() const { return outputParameter; }
105  OutputDataType& OutputParameter() { return outputParameter; }
106 
108  OutputDataType const& Delta() const { return delta; }
110  OutputDataType& Delta() { return delta; }
111 
113  OutputDataType const& Gradient() const { return gradient; }
115  OutputDataType& Gradient() { return gradient; }
116 
118  double const& Alpha() const { return alpha(0); }
120  double& Alpha() { return alpha(0); }
121 
125  template<typename Archive>
126  void serialize(Archive& ar, const unsigned int /* version */);
127 
128  private:
135  double Fn(const double x)
136  {
137  return std::max(x, alpha(0) * x);
138  }
139 
146  template<typename eT>
147  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
148  {
149  y = x;
150  arma::uvec negative = arma::find(x < 0);
151  y(negative) = x(negative) * alpha(0);
152  }
153 
160  double Deriv(const double x)
161  {
162  return (x >= 0) ? 1 : alpha(0);
163  }
164 
172  template<typename InputType, typename OutputType>
173  void Deriv(const InputType& x, OutputType& y)
174  {
175  y.set_size(arma::size(x));
176 
177  for (size_t i = 0; i < x.n_elem; i++)
178  {
179  y(i) = Deriv(x(i));
180  }
181  }
182 
184  OutputDataType delta;
185 
187  OutputDataType outputParameter;
188 
190  OutputDataType alpha;
191 
193  OutputDataType gradient;
194 
196  double userAlpha;
197 }; // class PReLU
198 
199 } // namespace ann
200 } // namespace mlpack
201 
202 // Include implementation.
203 #include "parametric_relu_impl.hpp"
204 
205 #endif
double & Alpha()
Modify the non zero gradient.
PReLU(const double userAlpha=0.03)
Create the PReLU object using the specified parameters.
.hpp
Definition: add_to_po.hpp:21
The core includes that mlpack expects; standard C++ includes and Armadillo.
OutputDataType const & Parameters() const
Get the parameters.
void Forward(const InputType &&input, OutputType &&output)
Ordinary feed forward pass of a neural network, evaluating the function f(x) by propagating the activ...
OutputDataType const & Gradient() const
Get the gradient.
The PReLU activation function, defined by (where alpha is trainable)
void Backward(const DataType &&input, DataType &&gy, DataType &&g)
Ordinary feed backward pass of a neural network, calculating the function f(x) by propagating x backw...
OutputDataType const & Delta() const
Get the delta.
OutputDataType & Parameters()
Modify the parameters.
void serialize(Archive &ar, const unsigned int)
Serialize the layer.
OutputDataType & OutputParameter()
Modify the output parameter.
OutputDataType & Delta()
Modify the delta.
OutputDataType & Gradient()
Modify the gradient.
OutputDataType const & OutputParameter() const
Get the output parameter.
double const & Alpha() const
Get the non zero gradient.