/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #include #ifdef PADDLE_TYPE_DOUBLE #define HL_FLOAT_MAX 3.40282347e+38F #define HL_FLOAT_MIN 1.17549435e-38F using real = double; #else #define HL_FLOAT_MAX 1.7976931348623157e+308 #define HL_FLOAT_MIN 2.2250738585072014e-308 using real = float; #endif /** * The maximum input value for exp, used to avoid overflow problem. * currently only used for tanh function. */ #define EXP_MAX_INPUT 40.0 /** * @brief DIVUP(x, y) is similar to ceil(x / y). * @note For CUDA, DIVUP will be used to specify * the size of blockDim. */ #ifndef DIVUP #define DIVUP(x, y) (((x) + (y)-1) / (y)) #endif /** * HPPL is an internal high performance parallel computing library * for high-level neural network routines, which can support many * heterogeneous compute architectures, such as GPU, FPGA, etc. */ /** * @brief HPPL CUDA Stream. * * @note Each thread can use HPPL_STREAM_* after calling hl_init. * HPPL_STREAM_DEFAULT is HPPL default stream. */ typedef enum { HPPL_STREAM_DEFAULT = 0, /* Thread Default Stream*/ HPPL_STREAM_1 = 1, HPPL_STREAM_2 = 2, HPPL_STREAM_3 = 3, HPPL_STREAM_4 = 4, HPPL_THREAD_STREAM_1 = 5, HPPL_THREAD_STREAM_2 = 6, HPPL_THREAD_STREAM_3 = 7, HPPL_THREAD_STREAM_4 = 8, HPPL_STREAM_END } hl_stream_t; /** * @brief HPPL activation mode. */ typedef enum { HL_ACTIVATION_SIGMOID = 0, HL_ACTIVATION_RELU = 1, HL_ACTIVATION_TANH = 2, HL_ACTIVATION_LINEAR = 3, HL_ACTIVATION_END } hl_activation_mode_t; /** * @brief Transpose type. */ typedef enum { HPPL_OP_N = 0, /* transpose */ HPPL_OP_T = 1, /* non transpose */ HPPL_OP_END } hl_trans_op_t; /** * @brief Lstm value. * * @param gateValue input value. * @param prevStateValue previous state value. * @param stateValue state value. * @param stateActiveValue state active value. * @param outputValue output value. */ typedef struct { real *gateValue; real *prevStateValue; real *stateValue; real *stateActiveValue; real *outputValue; real *checkIg; real *checkFg; real *checkOg; } hl_lstm_value; /** * @brief Lstm gradient. * * @param gateGrad input gradient. * @param prevStateGrad previous state gradient. * @param stateGrad state gradient. * @param stateActiveGrad state active gradient. * @param outputGrad output gradient. */ typedef struct { real *gateGrad; real *prevStateGrad; real *stateGrad; real *stateActiveGrad; real *outputGrad; real *checkIgGrad; real *checkFgGrad; real *checkOgGrad; } hl_lstm_grad; /** * @brief Gru value. * * @param gateWeight gate weight (updateGate + resetGate). * @param stateWeight frame state weight. * @param gateValue gate value results. * @param resetOutputValue resetOutput value. * @param outputValue output value. * @param prevOutValue previous output value. * */ typedef struct { real *gateWeight; real *stateWeight; real *gateValue; real *resetOutputValue; real *outputValue; real *prevOutValue; } hl_gru_value; /** * @brief Gru gradient. * * @param gateWeightGrad gate weight gradient. * @param stateWeightGrad frame state weight gradient. * @param gateGrad gate gradient results. * @param resetOutputGrad resetOutput gradient. * @param outputGrad output gradient. * @param prevOutGrad previous output gradient. */ typedef struct { real *gateWeightGrad; real *stateWeightGrad; real *gateGrad; real *resetOutputGrad; real *outputGrad; real *prevOutGrad; } hl_gru_grad; /** * @brief Sparse matrix value type. */ typedef enum { HL_NO_VALUE = 0, /* matrix values only 0 or 1 */ HL_FLOAT_VALUE = 1, HL_VALUE_END } hl_matrix_value_t; /** * @brief HPPL matrix format. */ typedef enum { HL_SPARSE_CSR = 0, HL_SPARSE_CSC = 1, HL_SPARSE_END } hl_matrix_format_t; typedef struct _hl_matrix_s *hl_matrix_s; /** * @brief HPPL sparse matrix. * * @param matrix sparse matrix. * @param format matrix format. * @param type the type of matrix values. * @param rows matrix rows. * @param cols matrix columns. * @param nnz nonzero values of sparse matrix. */ typedef struct { hl_matrix_s matrix; hl_matrix_format_t format; hl_matrix_value_t type; int rows; int cols; size_t nnz; } _hl_sparse_matrix_s, *hl_sparse_matrix_s; #ifdef __NVCC__ #include #include "paddle/legacy/cuda/include/hl_cuda.h" #include "paddle/utils/Logging.h" extern __thread bool g_sync_flag; extern __thread cudaStream_t default_stream; #define STREAM_DEFAULT default_stream /** * @brief Check cuda kernel execution. * @param msg error string */ #define CHECK_SYNC(msg) \ if (true == g_sync_flag) { \ hl_stream_synchronize(HPPL_STREAM_DEFAULT); \ cudaError_t err = (cudaError_t)hl_get_device_last_error(); \ CHECK_EQ(cudaSuccess, err) \ << "[" << msg << "] " \ << "CUDA error: " << hl_get_device_error_string((size_t)err); \ } // __shfl has been deprecated as of CUDA 9.0. #if CUDA_VERSION < 9000 template __forceinline__ __device__ T __shfl_down_sync(unsigned, T val, int delta) { return __shfl_down(val, delta); } template __forceinline__ __device__ T __shfl_sync(unsigned, T val, int src_line, int width) { return __shfl(val, src_line, width); } #define CREATE_SHFL_MASK(mask, predicate) mask = 0u; #else #define FULL_WARP_MASK 0xFFFFFFFF #define CREATE_SHFL_MASK(mask, predicate) \ mask = __ballot_sync(FULL_WARP_MASK, (predicate)) #endif #endif // __NVCC__