Neural Networks

Base

Functions

void hl_shrink_col2feature(const real *dataCol, size_t channels, size_t height, size_t width, size_t blockH, size_t blockW, size_t strideH, size_t strideW, size_t paddingH, size_t paddingW, size_t outputH, size_t outputW, real *dataIm, real alpha = 1.0f, real beta = 0.0f)

Shrink column to feature.

Parameters

• dataCol -

  expand data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• blockH -

  filter height.

• blockW -

  filter width.

• strideH -

  stride height.

• strideW -

  stride width.

• paddingH -

  padding height.

• paddingW -

  padding width.

• outputH -

  output height.

• outputW -

  output width.

• dataIm -

  output image data.

• alpha -

• beta -

void hl_expand_feature2col(const real *dataIm, size_t channels, size_t height, size_t width, size_t blockH, size_t blockW, size_t strideH, size_t strideW, size_t paddingH, size_t paddingW, size_t outputH, size_t outputW, real *dataCol)

Expand feature to column.

Parameters

• dataIm -

  input image data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• blockH -

  filter height.

• blockW -

  filter width.

• strideH -

  stride height.

• strideW -

  stride width.

• paddingH -

  padding height.

• paddingW -

  padding width.

• outputH -

  output height.

• outputW -

  output width.

• dataCol -

  expand data.

void hl_maxpool_forward(const int frameCnt, const real *inputData, const int channels, const int height, const int width, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, const int paddingH, const int paddingW, real *tgtData)

Maximum pool forward.

Parameters

• frameCnt -

  batch size of input image.

• inputData -

  input data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• pooledH -

  output image height.

• pooledW -

  output image width.

• sizeX -

  width of pooling window.

• sizeY -

  height of pooling window.

• strideH -

  pooling stride height.

• strideW -

  pooling stride width.

• paddingH -

  padding height.

• paddingW -

  padding width.

• tgtData -

  output data.

void hl_maxpool_backward(const int frameCnt, const real *inputData, const real *outData, const real *outGrad, const int channels, const int height, const int width, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, real *targetGrad)

Maximum pool backward.

Parameters

• frameCnt -

  batch size of input image.

• inputData -

  input data.

• outData -

  output data.

• outGrad -

  output grad data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• pooledH -

  output image height.

• pooledW -

  output image width.

• sizeX -

  width of pooling window.

• sizeY -

  height of pooling window.

• strideH -

  pooling stride height.

• strideW -

  pooling stride width.

• scaleA -

  scale.

• scaleB -

  scale.

• paddingH -

  padding height.

• paddingW -

  padding width.

• targetGrad -

  output grad.

void hl_avgpool_forward(const int frameCnt, const real *inputData, const int channels, const int height, const int width, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, const int paddingH, const int paddingW, real *tgtData)

Averge pool forward.

Parameters

• frameCnt -

  batch size of input image.

• inputData -

  input data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• pooledH -

  output image height.

• pooledW -

  output image width.

• sizeX -

  width of pooling window.

• sizeY -

  height of pooling window.

• strideH -

  pooling stride height.

• strideW -

  pooling stride width.

• paddingH -

  padding height.

• paddingW -

  padding width.

• tgtData -

  output data.

void hl_avgpool_backward(const int frameCnt, const real *outGrad, const int channels, const int height, const int width, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, int paddingH, int paddingW, real scaleA, real scaleB, real *backGrad)

Maximum pool backward.

Parameters

• frameCnt -

  batch size of input image.

• outGrad -

  input data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• pooledH -

  output image height.

• pooledW -

  output image width.

• sizeX -

  width of pooling window.

• sizeY -

  height of pooling window.

• strideH -

  pooling stride height.

• strideW -

  pooling stride width.

• paddingH -

  padding height.

• paddingW -

  padding width.

• scaleA -

  scale.

• scaleB -

  scale.

• backGrad -

  output grad.

void hl_CMRNorm_forward(size_t frameCnt, const real *in, real *scale, real *out, size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta)

Cross-map-respose normalize forward.

Parameters

• frameCnt -

  batch size of input image.

• in -

  input data.

• scale -

  buffer.

• out -

  output data.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• sizeX -

  size.

• alpha -

  scale.

• beta -

  scale.

void hl_CMRNorm_backward(size_t frameCnt, const real *inV, const real *scale, const real *outV, const real *outDiff, real *inDiff, size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta)

Cross-map-respose normalize backward.

Parameters

• frameCnt -

  batch size of input image.

• inV -

  input data.

• scale -

  buffer.

• outV -

  output value.

• outDiff -

  output grad.

• inDiff -

  input grad.

• channels -

  number of channel.

• height -

  image height.

• width -

  image width.

• sizeX -

  size.

• alpha -

  scale.

• beta -

  scale.

Defines

SIGMOID_THRESHOLD_MIN

sigmoid threshold maximum

SIGMOID_THRESHOLD_MAX

sigmoid threshold minimum

namespace hppl

Functions

real relu(const real a)

real sigmoid(const real a)

real tanh(const real a)

real linear(const real a)

real relu(const real a, const real b)

real sigmoid(const real a, const real b)

real tanh(const real a, const real b)

real linear(const real a, const real b)

namespace hppl

Functions

__m256 relu(const __m256 a)

__m256 sigmoid(const __m256 a)

__m256 tanh(const __m256 a)

__m256 linear(const __m256 a)

__m256 relu(const __m256 a, const __m256 b)

__m256 sigmoid(const __m256 a, const __m256 b)

__m256 tanh(const __m256 a, const __m256 b)

__m256 linear(const __m256 a, const __m256 b)

Defines

HL_DEVICE_FUNCTIONS_CUH_

namespace paddle

Functions

template <class T>

__device__ T paddle::paddleAtomicAdd(T * address, T val)

template <>

__device__ float paddle::paddleAtomicAdd(float * address, float val)

template <>

__device__ double paddle::paddleAtomicAdd(double * address, double val)

Defines

HL_GPU_FUNCTIONS_CUH_

Activation Functions

Defines

HPPL_ACTIVE_FUNCTION

Active functions: sigmoid, relu, tanh and linear.

namespace hppl

template <class T>

class Active

#include <hl_activation_functions.h>

Hppl supports sigmoid, relu, tanh, linear active functions for neural networks’ forward and backward activation.

Public Types

typedef >

typedef T (*hppl::Active<T>::forward)(T)

typedef >

typedef T (*hppl::Active<T>::backward)(T, T)

namespace cpu

Variables

Active<real>::forward forward[]

Active<real>::backward backward[]

RNN Related APIs

Defines

HL_RECURRENT_APPLY_CUH_

Functions

template <class Op>

void hl_cpu_lstm_forward(Op op, hl_lstm_value value, int frameSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Cpu lstm forward one sequence.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  hl_lstm_value type.

• frameSize -

  frame size.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

template <class Op>

void hl_cpu_lstm_backward(Op op, hl_lstm_value value, hl_lstm_grad grad, int frameSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Cpu lstm backward one sequence.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  lstm value.

• grad -

  output gradient.

• frameSize -

  frame size.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

template <class Op>

void hl_gpu_lstm_forward(Op op, hl_lstm_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Gpu lstm batch forward.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  lstm value.

• frameSize -

  frame size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

template <class Op>

void hl_gpu_lstm_backward(Op op, hl_lstm_value value, hl_lstm_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Gpu lstm batch backward.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  lstm value.

• grad -

  lstm gradient.

• frameSize -

  frame size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

template <class OpResetOutput, class OpFinalOutput>

void hl_cpu_gru_forward(OpResetOutput opResetOutput, OpFinalOutput opFinalOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Cpu gru forward.

Parameters

• opResetOutput -

  hl_gru_ops.cuh

• opFinalOutput -

  hl_gru_ops.cuh

• value -

  gru value.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

template <class OpStateGrad, class OpResetGrad>

void hl_cpu_gru_backward(OpStateGrad opStateGrad, OpResetGrad opResetGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Cpu gru forward.

Parameters

• opStateGrad -

  hl_gru_ops.cuh

• opResetGrad -

  hl_gru_ops.cuh

• value -

  gru value.

• grad -

  gru gradient.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

template <class OpResetOutput, class OpFinalOutput>

void hl_gpu_gru_forward(OpResetOutput opResetOutput, OpFinalOutput opFinalOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Gpu gru forward.

Parameters

• opResetOutput -

  hl_gru_ops.cuh

• opFinalOutput -

  hl_gru_ops.cuh

• value -

  gru value.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

template <class OpStateGrad, class OpResetGrad>

void hl_gpu_gru_backward(OpStateGrad opStateGrad, OpResetGrad opResetGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Gpu gru forward.

Parameters

• opStateGrad -

  hl_gru_ops.cuh

• opResetGrad -

  hl_gru_ops.cuh

• value -

  gru value.

• grad -

  gru gradient.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

Functions

void hl_max_sequence_forward(real *input, const int *sequence, real *output, int *index, int numSequences, int dim)

Maximum sequence forward.

Parameters

• input -

  each sequence contains some instances.

• sequence -

  sequence index..

• output -

  max instance in this sequence.

• index -

  index of max instance.

• numSequences -

  size of sequence[in].

• dim -

  input dimension.

void hl_max_sequence_backward(real *outputGrad, int *index, real *inputGrad, int numSequences, int dim)

Maximum sequence backward.

Parameters

• outputGrad -

  output gradient.

• index -

  index of max instance.

• inputGrad -

  input gradient.

• numSequences -

  size of sequence[in].

• dim -

  input dimension.

void hl_context_projection_forward(real *input, const int *sequence, real *weightData, real *output, int numSequences, int inputDim, int contextLength, int contextStart, int beginPad, bool isPadding)

Context projection forward.

Parameters

• input -

  input sequence.

• sequence -

  sequence index.

• weightData -

  padding data.

• output -

  output sequence.

• numSequences -

  number of sequences.

• inputDim -

  input sequence dimension.

• contextLength -

  context length.

• contextStart -

  context start.

• beginPad -

  number of extra timesteps added at the beginning.

• isPadding -

  trainable padding.

void hl_context_projection_backward_data(real *outputGrad, const int *sequence, real *inputGrad, int numSequences, int inputDim, int contextLength, int contextStart)

Context projection backward data.

Parameters

• outputGrad -

  output gradient.

• sequence -

  sequence index.

• inputGrad -

  input gradient.

• numSequences -

  number of sequences.

• inputDim -

  input sequence dimension.

• contextLength -

  context length.

• contextStart -

  context start.

void hl_context_projection_backward_weight(real *outputGrad, const int *sequence, real *weightGrad, int numSequences, int weightDim, int totalPad, int contextLength, int contextStart, int beginPad)

Context projection backward weight.

Parameters

• outputGrad -

  output gradient.

• sequence -

  sequence index.

• weightGrad -

  weight gradient.

• numSequences -

  number of sequences.

• weightDim -

  input sequence dimension.

• totalPad -

  number of extra timesteps.

• contextLength -

  context length.

• contextStart -

  context start.

• beginPad -

  number of extra timesteps added at the beginning.

void hl_sequence2batch_copy(real *batch, real *sequence, int *batchIndex, int seqWidth, int batchCount, bool seq2batch)

Memory copy from sequence to batch.

if seq2batch == true

copy from sequence to batch: batch[i] = sequence[batchIndex[i]].

if seq2batch == false

copy from batch to sequence: sequence[batchIndex[i]] = batch[i].

Parameters

• batch -

  batch matrix.

• sequence -

  equence matrix.

• batchIndex -

  index vector.

• seqWidth -

  width of sequence.

• batchCount -

  number of batchIndex.

• seq2batch -

  copy direction.

void hl_sequence2batch_add(real *batch, real *sequence, int *batchIndex, int seqWidth, int batchCount, bool seq2batch)

Add sequence to batch.

if seq2batch == true

add sequence to batch: batch[i] = sequence[batchIndex[i]].

if seq2batch == false

add batch to sequence: sequence[batchIndex[i]] = batch[i].

Parameters

• batch -

  batch matrix.

• sequence -

  equence matrix.

• batchIndex -

  index vector.

• seqWidth -

  width of sequence.

• batchCount -

  number of batchIndex.

• seq2batch -

  copy direction.

void hl_sequence_avg_forward(real *dst, real *src, const int *starts, int height, int width, const int mode)

dst = Op(src), src is sequence.

mode = 0, Op is average.

mode = 1, Op is sum.

mode = 2, Op is sum(src)/sqrt(N), N is sequence length.

Parameters

• dst -

  destination data.

• src -

  source data.

• starts -

  sequence start positions.

• height -

  height of dst data.

• width -

  width of dst data.

• mode -

  0: avreage, 1: sum, 2: divide by square root of sequenceLength

LSTM Model

Functions

void hl_lstm_parallel_forward(real *gateValue, real *stateValue, real *preOutputValue, real *outputValue, real *checkIg, real *checkFg, real *checkOg, real *weight, const int *sequence, int frameSize, int numSequences, bool reversed, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Lstm sequence parallel forward.

Note

Only support frameSize = 32 or 64.

Parameters

• gateValue -

  input value.

• stateValue -

  state value.

• preOutputValue -

  prev output value.

• outputValue -

  output value.

• checkIg -

  bias.

• checkFg -

  bias.

• checkOg -

  bias.

• weight -

  weight.

• sequence -

  sequence index.

• frameSize -

  frame size.

• numSequences -

  number of sequences.

• reversed -

  reverse.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

void hl_lstm_parallel_backward_data(real *gateValue, real *gateGrad, real *stateValue, real *stateGrad, real *preOutputValue, real *preOutputGrad, real *outputGrad, real *checkIg, real *checkIgGrad, real *checkFg, real *checkFgGrad, real *checkOg, real *checkOgGrad, real *weight, const int *sequence, int frameSize, int numSequences, bool reversed, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Lstm sequence parallel backward data.

Note

Only support frameSize = 32 or 64.

Parameters

• gateValue -

  input value.

• gateGrad -

  input gradient.

• stateValue -

  state value.

• stateGrad -

  state gradient.

• preOutputValue -

  prev output value.

• preOutputGrad -

  prev output gradient.

• outputGrad -

  output gradient.

• checkIg -

  bias.

• checkIgGrad -

  bias gradient.

• checkFg -

  bias.

• checkFgGrad -

  bias gradient.

• checkOg -

  bias.

• checkOgGrad -

  bias gradient.

• weight -

  weight.

• sequence -

  sequence index.

• frameSize -

  frame size.

• numSequences -

  number of sequences.

• reversed -

  reverse.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

void hl_lstm_parallel_backward_weight(real *weightGrad, real *outputValue, real *gateGrad, const int *sequence, int frameSize, int batchSize, int numSequences, bool reversed)

Lstm sequence parallel backward weight.

Parameters

• weightGrad -

  weight gradient.

• outputValue -

  output value.

• gateGrad -

  gate gradient.

• sequence -

  sequence index.

• frameSize -

  frame size.

• batchSize -

  batch size.

• numSequences -

  number of sequences.

• reversed -

  reverse.

Defines

HL_GPU_LSTM_CUH_

Functions

template <class Op>

void hl_gpu_lstm_forward(Op op, hl_lstm_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Gpu lstm batch forward.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  lstm value.

• frameSize -

  frame size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

template <class Op>

void hl_gpu_lstm_backward(Op op, hl_lstm_value value, hl_lstm_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate, hl_activation_mode_t active_state)

Gpu lstm batch backward.

Parameters

• op -

  hl_lstm_ops.cuh

• value -

  lstm value.

• grad -

  lstm gradient.

• frameSize -

  frame size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

• active_state -

  actvie gate type.

Defines

HL_LSTM_OPS_CUH_

INLINE

namespace hppl

namespace backward

class lstm

Public Functions

INLINE void hppl::backward::lstm::operator()(real & valueIn, real & valueIg, real & valueFg, real & valueOg, real & gradIn, real & gradIg, real & gradFg, real & gradOg, real & prevState, real & prevStateGrad, real & state, real & stateGrad, real & stateAtv, real & outputGrad, real & checkI, real & checkF, real & checkO, real & checkIGrad, real & checkFGrad, real & checkOGrad, Active < real >::backward actInput, Active < real >::backward actGate, Active < real >::backward actState)

Parameters

• valueIn -

  input

• valueIg -

  input gate

• valueFg -

  forget gate

• valueOg -

  output gate

• gradIn -

  input grad

• gradIg -

  input gate grad

• gradFg -

  forget gate grad

• gradOg -

  output gate grad

• prevState -

  previous state value

• prevStateGrad -

  previous state grad

• state -

  current state value

• stateGrad -

  current state grad

• stateAtv -

  state active

• outputGrad -

  output grad

• checkI -

  check input gate

• checkF -

  check forget gate

• checkO -

  check output gate

• checkIGrad -

  check input gate grad

• checkFGrad -

  check forget gate grad

• checkOGrad -

  check output gate grad

• actInput -

  backward function of input

• actGate -

  backward function of gate

• actState -

  backward function of state

Public Static Attributes

const bool avx

namespace forward

class lstm

Public Functions

INLINE void hppl::forward::lstm::operator()(real & valueIn, real & valueIg, real & valueFg, real & valueOg, real & prevState, real & state, real & stateAtv, real & output, real & checkI, real & checkF, real & checkO, Active < real >::forward actInput, Active < real >::forward actGate, Active < real >::forward actState)

Parameters

• valueIn -

  input

• valueIg -

  input gate

• valueFg -

  forget gate

• valueOg -

  output gate

• prevState -

  previous state

• state -

  current state

• stateAtv -

  state active

• output -

  output

• checkI -

  check input gate

• checkF -

  check forget gate

• checkO -

  check output gate

• actInput -

  forward function of input

• actGate -

  forward function of gate

• actState -

  forward function of state

Public Static Attributes

const bool avx

GRU Model

Defines

HL_GRU_OPS_CUH_

INLINE

namespace hppl

namespace backward

class gru_stateGrad

Public Functions

INLINE void hppl::backward::gru_stateGrad::operator()(real & valueUpdateGate, real & gradUpdateGate, real & valueFrameState, real & gradFrameState, real & valuePrevOut, real & gradPrevOut, real & gradOutput, Active < real >::backward actInput)

Parameters

• valueUpdateGate -

  update gate value

• gradUpdateGate -

  update gate grad

• valueFrameState -

  frame state value

• gradFrameState -

  frame state grad

• valuePrevOut -

  previous output value

• gradPrevOut -

  previous output grad

• gradOutput -

  output grad

• actInput -

  backward function of frame state

Public Static Attributes

const bool avx

class gru_resetGrad

Public Functions

INLINE void hppl::backward::gru_resetGrad::operator()(real & valueUpdateGate, real & gradUpdateGate, real & valueResetGate, real & gradResetGate, real & valuePrevOut, real & gradPrevOut, real & gradResetOutput, Active < real >::backward actGate)

Parameters

• valueUpdateGate -

  update gate value

• gradUpdateGate -

  update gate grad

• valueResetGate -

  reset gate value

• gradResetGate -

  reset gate grad

• valuePrevOut -

  previous output value

• gradPrevOut -

  previous output grad

• gradResetOutput -

  reset output grad (temp val)

• actGate -

  backward function of gate

Public Static Attributes

const bool avx

namespace forward

class gru_resetOutput

Public Functions

INLINE void hppl::forward::gru_resetOutput::operator()(real & valueUpdateGate, real & valueResetGate, real & prevOut, real & valueResetOutput, Active < real >::forward actGate)

Parameters

• valueUpdateGate -

  update gate

• valueResetGate -

  reset gate

• prevOut -

  previous output

• valueResetOutput -

  intermediate value for frame state

• actGate -

  forward function of gate

Public Static Attributes

const bool avx

class gru_finalOutput

Public Functions

INLINE void hppl::forward::gru_finalOutput::operator()(real & valueUpdateGate, real & valueFrameState, real & prevOut, real & valueOutput, Active < real >::forward actInput)

Parameters

• valueUpdateGate -

  update gate

• valueFrameState -

  frame state ({{h}_t})

• prevOut -

  previous output

• valueOutput -

  output

• actInput -

  forward function of node

Public Static Attributes

const bool avx

Defines

HL_CPU_GRU_CUH_

CBLAS_GEMM

Functions

template <class OpResetOutput>

void hl_naive_gru_forward_reset_output(OpResetOutput opResetOutput, real *gateValue, real *resetOutputValue, real *prevOutputValue, int frameSize, hl_activation_mode_t active_gate)

template <class OpFinalOutput>

void hl_naive_gru_forward_final_output(OpFinalOutput opFinalOutput, real *gateValue, real *prevOutputValue, real *outputValue, int frameSize, hl_activation_mode_t active_node)

template <class OpResetOutput>

void hl_avx_gru_forward_reset_output(OpResetOutput opResetOutput, real *gateValue, real *resetOutputValue, real *prevOutputValue, int frameSize, hl_activation_mode_t active_gate)

template <class OpFinalOutput>

void hl_avx_gru_forward_final_output(OpFinalOutput opFinalOutput, real *gateValue, real *prevOutputValue, real *outputValue, int frameSize, hl_activation_mode_t active_node)

template <class OpResetOutput>

void forward_reset_output(OpResetOutput opResetOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_gate)

template <class OpFinalOutput>

void forward_final_output(OpFinalOutput opFinalOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_node)

template <class OpResetOutput, class OpFinalOutput>

void hl_cpu_gru_forward(OpResetOutput opResetOutput, OpFinalOutput opFinalOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Cpu gru forward.

Parameters

• opResetOutput -

  hl_gru_ops.cuh

• opFinalOutput -

  hl_gru_ops.cuh

• value -

  gru value.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

template <class OpStateGrad>

void hl_naive_gru_backward_state_grad(OpStateGrad opStateGrad, real *gateValue, real *gateGrad, real *prevOutValue, real *prevOutGrad, real *outputGrad, int frameSize, hl_activation_mode_t active_node)

template <class OpResetGrad>

void hl_naive_gru_backward_reset_grad(OpResetGrad opResetGrad, real *gateValue, real *gateGrad, real *prevOutValue, real *prevOutGrad, real *resetOutputGrad, int frameSize, hl_activation_mode_t active_gate)

template <class OpStateGrad>

void hl_avx_gru_backward_state_grad(OpStateGrad opStateGrad, real *gateValue, real *gateGrad, real *prevOutValue, real *prevOutGrad, real *outputGrad, int frameSize, hl_activation_mode_t active_node)

template <class OpResetGrad>

void hl_avx_gru_backward_reset_grad(OpResetGrad opResetGrad, real *gateValue, real *gateGrad, real *prevOutValue, real *prevOutGrad, real *resetOutputGrad, int frameSize, hl_activation_mode_t active_gate)

template <class OpStateGrad>

void backward_state_grad(OpStateGrad opStateGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node)

template <class OpResetGrad>

void backward_reset_grad(OpResetGrad opResetGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_gate)

template <class OpStateGrad, class OpResetGrad>

void hl_cpu_gru_backward(OpStateGrad opStateGrad, OpResetGrad opResetGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Cpu gru forward.

Parameters

• opStateGrad -

  hl_gru_ops.cuh

• opResetGrad -

  hl_gru_ops.cuh

• value -

  gru value.

• grad -

  gru gradient.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

Defines

HL_GPU_GRU_CUH_

Functions

template <class OpResetOutput, class OpFinalOutput>

void hl_gpu_gru_forward(OpResetOutput opResetOutput, OpFinalOutput opFinalOutput, hl_gru_value value, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Gpu gru forward.

Parameters

• opResetOutput -

  hl_gru_ops.cuh

• opFinalOutput -

  hl_gru_ops.cuh

• value -

  gru value.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.

template <class OpStateGrad, class OpResetGrad>

void hl_gpu_gru_backward(OpStateGrad opStateGrad, OpResetGrad opResetGrad, hl_gru_value value, hl_gru_grad grad, int frameSize, int batchSize, hl_activation_mode_t active_node, hl_activation_mode_t active_gate)

Gpu gru forward.

Parameters

• opStateGrad -

  hl_gru_ops.cuh

• opResetGrad -

  hl_gru_ops.cuh

• value -

  gru value.

• grad -

  gru gradient.

• frameSize -

  frame length/size.

• batchSize -

  size of current batch.

• active_node -

  active input type.

• active_gate -

  active state type.