leaky_relu.hpp
Go to the documentation of this file.
1 
14 #ifndef MLPACK_METHODS_ANN_LAYER_LEAKYRELU_HPP
15 #define MLPACK_METHODS_ANN_LAYER_LEAKYRELU_HPP
16 
17 #include <mlpack/prereqs.hpp>
18 
19 namespace mlpack {
20 namespace ann {
21 
40 template <
41  typename InputDataType = arma::mat,
42  typename OutputDataType = arma::mat
43 >
44 class LeakyReLU
45 {
46  public:
54  LeakyReLU(const double alpha = 0.03);
55 
63  template<typename InputType, typename OutputType>
64  void Forward(const InputType&& input, OutputType&& output);
65 
75  template<typename DataType>
76  void Backward(const DataType&& input, DataType&& gy, DataType&& g);
77 
79  OutputDataType const& OutputParameter() const { return outputParameter; }
81  OutputDataType& OutputParameter() { return outputParameter; }
82 
84  OutputDataType const& Delta() const { return delta; }
86  OutputDataType& Delta() { return delta; }
87 
89  double const& Alpha() const { return alpha; }
91  double& Alpha() { return alpha; }
92 
96  template<typename Archive>
97  void serialize(Archive& ar, const unsigned int /* version */);
98 
99  private:
106  double Fn(const double x)
107  {
108  return std::max(x, alpha * x);
109  }
110 
117  template<typename eT>
118  void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
119  {
120  y = arma::max(x, alpha * x);
121  }
122 
129  double Deriv(const double x)
130  {
131  return (x >= 0) ? 1 : alpha;
132  }
133 
141  template<typename InputType, typename OutputType>
142  void Deriv(const InputType& x, OutputType& y)
143  {
144  y.set_size(arma::size(x));
145 
146  for (size_t i = 0; i < x.n_elem; i++)
147  {
148  y(i) = Deriv(x(i));
149  }
150  }
151 
153  OutputDataType delta;
154 
156  OutputDataType outputParameter;
157 
159  double alpha;
160 }; // class LeakyReLU
161 
162 } // namespace ann
163 } // namespace mlpack
164 
165 // Include implementation.
166 #include "leaky_relu_impl.hpp"
167 
168 #endif
void serialize(Archive &ar, const unsigned int)
Serialize the layer.
LeakyReLU(const double alpha=0.03)
Create the LeakyReLU object using the specified parameters.
.hpp
Definition: add_to_po.hpp:21
void Backward(const DataType &&input, DataType &&gy, DataType &&g)
Ordinary feed backward pass of a neural network, calculating the function f(x) by propagating x backw...
The core includes that mlpack expects; standard C++ includes and Armadillo.
void Forward(const InputType &&input, OutputType &&output)
Ordinary feed forward pass of a neural network, evaluating the function f(x) by propagating the activ...
The LeakyReLU activation function, defined by.
Definition: leaky_relu.hpp:44
OutputDataType const & Delta() const
Get the delta.
Definition: leaky_relu.hpp:84
double const & Alpha() const
Get the non zero gradient.
Definition: leaky_relu.hpp:89
OutputDataType const & OutputParameter() const
Get the output parameter.
Definition: leaky_relu.hpp:79
double & Alpha()
Modify the non zero gradient.
Definition: leaky_relu.hpp:91
OutputDataType & OutputParameter()
Modify the output parameter.
Definition: leaky_relu.hpp:81
OutputDataType & Delta()
Modify the delta.
Definition: leaky_relu.hpp:86