From ef6ea790dc91d55bfedfc09c56e87de94a231bea Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 May 2018 13:30:50 +0800 Subject: [PATCH] Clean and extract blas --- .../operators/bilinear_tensor_product_op.h | 2 +- paddle/fluid/operators/conv_op.h | 2 +- paddle/fluid/operators/conv_transpose_op.h | 2 +- paddle/fluid/operators/gru_unit_op.h | 5 +- paddle/fluid/operators/layer_norm_op.h | 14 +- paddle/fluid/operators/lstm_op.h | 2 +- paddle/fluid/operators/lstmp_op.h | 7 +- paddle/fluid/operators/math/CMakeLists.txt | 3 +- paddle/fluid/operators/math/blas.cc | 22 +++ paddle/fluid/operators/math/blas.h | 152 ++++++++++++++++++ paddle/fluid/operators/math/blas_impl.cu.h | 87 ++++++++++ paddle/fluid/operators/math/blas_impl.h | 89 +++++++++- paddle/fluid/operators/math/context_project.h | 11 +- paddle/fluid/operators/math/gru_compute.cc | 2 +- paddle/fluid/operators/math/gru_compute.cu | 2 +- paddle/fluid/operators/math/math_function.cc | 127 --------------- paddle/fluid/operators/math/math_function.cu | 145 +---------------- paddle/fluid/operators/math/math_function.h | 93 ----------- .../operators/math/math_function_test.cc | 6 +- .../operators/math/math_function_test.cu | 6 +- paddle/fluid/operators/math/matmul.h | 15 +- paddle/fluid/operators/mul_op.h | 4 +- 22 files changed, 398 insertions(+), 400 deletions(-) create mode 100644 paddle/fluid/operators/math/blas.cc create mode 100644 paddle/fluid/operators/math/blas.h diff --git a/paddle/fluid/operators/bilinear_tensor_product_op.h b/paddle/fluid/operators/bilinear_tensor_product_op.h index 7191711a7..f23336f7b 100644 --- a/paddle/fluid/operators/bilinear_tensor_product_op.h +++ b/paddle/fluid/operators/bilinear_tensor_product_op.h @@ -16,7 +16,7 @@ limitations under the License. */ #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/operators/math/blas.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/conv_op.h b/paddle/fluid/operators/conv_op.h index 819d678fd..c51898abb 100644 --- a/paddle/fluid/operators/conv_op.h +++ b/paddle/fluid/operators/conv_op.h @@ -17,9 +17,9 @@ limitations under the License. */ #include #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/depthwise_conv.h" #include "paddle/fluid/operators/math/im2col.h" -#include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/vol2col.h" namespace paddle { diff --git a/paddle/fluid/operators/conv_transpose_op.h b/paddle/fluid/operators/conv_transpose_op.h index 353f004b5..9276e5bfe 100644 --- a/paddle/fluid/operators/conv_transpose_op.h +++ b/paddle/fluid/operators/conv_transpose_op.h @@ -16,8 +16,8 @@ limitations under the License. */ #include #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/im2col.h" -#include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/vol2col.h" namespace paddle { diff --git a/paddle/fluid/operators/gru_unit_op.h b/paddle/fluid/operators/gru_unit_op.h index 49e657a27..2d9faed64 100644 --- a/paddle/fluid/operators/gru_unit_op.h +++ b/paddle/fluid/operators/gru_unit_op.h @@ -14,11 +14,10 @@ limitations under the License. */ #pragma once -#include "paddle/fluid/operators/activation_op.h" -#include "paddle/fluid/operators/math/math_function.h" - #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/activation_op.h" +#include "paddle/fluid/operators/math/blas.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/layer_norm_op.h b/paddle/fluid/operators/layer_norm_op.h index 7b84ba0a7..2e54bb497 100644 --- a/paddle/fluid/operators/layer_norm_op.h +++ b/paddle/fluid/operators/layer_norm_op.h @@ -15,8 +15,8 @@ limitations under the License. */ #pragma once #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" - #include "paddle/fluid/operators/elementwise_op_function.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/math_function.h" namespace paddle { @@ -46,9 +46,9 @@ class RowwiseMean2D { } void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { - math::gemv( - context, false, left_, right_, 1., input.data(), divisor_.data(), - 0., out->data()); + math::GetBlas(context).GEMV( + false, left_, right_, 1., input.data(), divisor_.data(), 0., + out->data()); } private: @@ -93,9 +93,9 @@ class ColwiseSum2D { void operator()(const platform::CUDADeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { - math::gemv( - context, true, left_, right_, 1., input.data(), divisor_.data(), - 0., out->data()); + math::GetBlas(context).GEMV( + true, left_, right_, 1., input.data(), divisor_.data(), 0., + out->data()); } private: diff --git a/paddle/fluid/operators/lstm_op.h b/paddle/fluid/operators/lstm_op.h index 382be6598..7d62d2d02 100644 --- a/paddle/fluid/operators/lstm_op.h +++ b/paddle/fluid/operators/lstm_op.h @@ -15,9 +15,9 @@ limitations under the License. */ #pragma once #include #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/detail/activation_functions.h" #include "paddle/fluid/operators/math/lstm_compute.h" -#include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/sequence2batch.h" namespace paddle { diff --git a/paddle/fluid/operators/lstmp_op.h b/paddle/fluid/operators/lstmp_op.h index 557ad3991..370dd04d1 100644 --- a/paddle/fluid/operators/lstmp_op.h +++ b/paddle/fluid/operators/lstmp_op.h @@ -14,15 +14,14 @@ limitations under the License. */ #pragma once #include +#include "paddle/fluid/framework/eigen.h" +#include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/activation_op.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/detail/activation_functions.h" #include "paddle/fluid/operators/math/lstm_compute.h" -#include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/sequence2batch.h" -#include "paddle/fluid/framework/eigen.h" -#include "paddle/fluid/framework/op_registry.h" - namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt index ee0e91132..f36e9444d 100644 --- a/paddle/fluid/operators/math/CMakeLists.txt +++ b/paddle/fluid/operators/math/CMakeLists.txt @@ -41,7 +41,8 @@ math_library(depthwise_conv) math_library(gru_compute DEPS activation_functions math_function) math_library(im2col) math_library(lstm_compute DEPS activation_functions) -math_library(math_function DEPS cblas) +cc_library(blas SRCS blas.cc DEPS cblas framework_proto) +math_library(math_function DEPS blas) math_library(maxouting) math_library(pooling) math_library(selected_rows_functor DEPS selected_rows math_function) diff --git a/paddle/fluid/operators/math/blas.cc b/paddle/fluid/operators/math/blas.cc new file mode 100644 index 000000000..3eeb77546 --- /dev/null +++ b/paddle/fluid/operators/math/blas.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/operators/math/blas.h" +namespace paddle { +namespace operators { +namespace math { +// Do nothing. Blas is a header only library. +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/math/blas.h b/paddle/fluid/operators/math/blas.h new file mode 100644 index 000000000..5cd2f855d --- /dev/null +++ b/paddle/fluid/operators/math/blas.h @@ -0,0 +1,152 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/framework/tensor.h" + +#ifdef PADDLE_WITH_MKLML +#include +#include +#include +#endif + +#ifdef PADDLE_USE_OPENBLAS +#include +#include +#endif + +#ifndef LAPACK_FOUND +extern "C" { +#include // NOLINT +int LAPACKE_sgetrf(int matrix_layout, int m, int n, float* a, int lda, + int* ipiv); +int LAPACKE_dgetrf(int matrix_layout, int m, int n, double* a, int lda, + int* ipiv); +int LAPACKE_sgetri(int matrix_layout, int n, float* a, int lda, + const int* ipiv); +int LAPACKE_dgetri(int matrix_layout, int n, double* a, int lda, + const int* ipiv); +} +#endif + +namespace paddle { +namespace operators { +namespace math { + +template +class Blas { + public: + explicit Blas(const DeviceContext& context) : context_(context) {} + + template + void GEMM(CBLAS_TRANSPOSE transA, CBLAS_TRANSPOSE transB, int M, int N, int K, + T alpha, const T* A, const T* B, T beta, T* C) const; + + template + void GEMM(bool transA, bool transB, int M, int N, int K, T alpha, const T* A, + int lda, const T* B, int ldb, T beta, T* C, int ldc) const; + + template + void MatMul(const framework::Tensor& mat_a, bool trans_a, + const framework::Tensor& mat_b, bool trans_b, T alpha, + framework::Tensor* mat_out, T beta) const; + + template + void MatMul(const framework::Tensor& mat_a, bool trans_a, + const framework::Tensor& mat_b, bool trans_b, + framework::Tensor* mat_out) const { + MatMul(mat_a, trans_a, mat_b, trans_b, static_cast(1.0), mat_out, + static_cast(0.0)); + } + + template + void MatMul(const framework::Tensor& mat_a, const framework::Tensor& mat_b, + framework::Tensor* mat_out) const { + this->template MatMul(mat_a, false, mat_b, false, mat_out); + } + + template + void AXPY(int n, T alpha, const T* x, T* y) const; + + template + void GEMV(bool trans_a, int M, int N, T alpha, const T* A, const T* B, T beta, + T* C) const; + + template + void BatchedGEMM(CBLAS_TRANSPOSE transA, CBLAS_TRANSPOSE transB, int M, int N, + int K, T alpha, const T* A, const T* B, T beta, T* C, + int batchCount, int64_t strideA, int64_t strideB) const; + + private: + const DeviceContext& context_; +}; + +template +class BlasT : private Blas { + public: + using Blas::Blas; + + template + void GEMM(ARGS... args) const { + Base()->template GEMM(args...); + } + + template + void MatMul(ARGS... args) const { + Base()->template MatMul(args...); + } + + template + void AXPY(ARGS... args) const { + Base()->template AXPY(args...); + } + + template + void GEMV(ARGS... args) const { + Base()->template GEMV(args...); + } + + template + void BatchedGEMM(ARGS... args) const { + Base()->template BatchedGEMM(args...); + } + + private: + const Blas* Base() const { + return static_cast*>(this); + } +}; + +template +inline BlasT GetBlas( + const framework::ExecutionContext& exe_ctx) { + return BlasT( + exe_ctx.template device_context()); +} + +template +inline BlasT GetBlas(const DeviceContext& dev_ctx) { + return BlasT(dev_ctx); +} + +} // namespace math +} // namespace operators +} // namespace paddle + +#include "paddle/fluid/operators/math/blas_impl.h" +#ifdef PADDLE_WITH_CUDA +#include "paddle/fluid/operators/math/blas_impl.cu.h" +#endif diff --git a/paddle/fluid/operators/math/blas_impl.cu.h b/paddle/fluid/operators/math/blas_impl.cu.h index ad2835af0..c76fc17d7 100644 --- a/paddle/fluid/operators/math/blas_impl.cu.h +++ b/paddle/fluid/operators/math/blas_impl.cu.h @@ -30,6 +30,25 @@ struct CUBlas { static void GEMM(ARGS... args) { PADDLE_ENFORCE(platform::dynload::cublasSgemm(args...)); } + + template + static void AXPY(ARGS... args) { + PADDLE_ENFORCE(platform::dynload::cublasSaxpy(args...)); + } + + template + static void GEMV(ARGS... args) { + PADDLE_ENFORCE(platform::dynload::cublasSgemv(args...)); + } + + template + static void GEMM_BATCH(ARGS... args) { +#if CUDA_VERSION >= 8000 + PADDLE_ENFORCE(platform::dynload::cublasSgemmStridedBatched(args...)); +#else + PADDLE_THROW("SgemmStridedBatched is not supported on cuda <= 7.5"); +#endif + } }; template <> @@ -38,6 +57,25 @@ struct CUBlas { static void GEMM(ARGS... args) { PADDLE_ENFORCE(platform::dynload::cublasDgemm(args...)); } + + template + static void AXPY(ARGS... args) { + PADDLE_ENFORCE(platform::dynload::cublasDaxpy(args...)); + } + + template + static void GEMV(ARGS... args) { + PADDLE_ENFORCE(platform::dynload::cublasDgemv(args...)); + } + + template + static void GEMM_BATCH(ARGS... args) { +#if CUDA_VERSION >= 8000 + PADDLE_ENFORCE(platform::dynload::cublasDgemmStridedBatched(args...)); +#else + PADDLE_THROW("DgemmStridedBatched is not supported on cuda <= 7.5"); +#endif + } }; template <> @@ -57,6 +95,15 @@ struct CUBlas { reinterpret_cast(beta), reinterpret_cast<__half *>(C), ldc)); } + + template + static void GEMM_BATCH(ARGS... args) { +#if CUDA_VERSION >= 8000 + PADDLE_ENFORCE(platform::dynload::cublasHgemmStridedBatched(args...)); +#else + PADDLE_THROW("HgemmStridedBatched is not supported on cuda <= 7.5"); +#endif + } }; template <> @@ -144,6 +191,46 @@ void Blas::GEMM(bool transA, bool transB, int M, B, ldb, A, lda, &beta, C, ldc); } +template <> +template +void Blas::AXPY(int n, T alpha, const T *x, + T *y) const { + CUBlas::AXPY(context_.cublas_handle(), n, &alpha, x, 1, y, 1); +} + +template <> +template +void Blas::GEMV(bool trans_a, int M, int N, + T alpha, const T *A, const T *B, + T beta, T *C) const { + cublasOperation_t cuTransA = !trans_a ? CUBLAS_OP_T : CUBLAS_OP_N; + + CUBlas::GEMV(context_.cublas_handle(), cuTransA, N, M, &alpha, A, N, B, 1, + &beta, C, 1); +} + +template <> +template +void Blas::BatchedGEMM( + CBLAS_TRANSPOSE transA, CBLAS_TRANSPOSE transB, int M, int N, int K, + T alpha, const T *A, const T *B, T beta, T *C, int batchCount, + int64_t strideA, int64_t strideB) const { + // Note that cublas follows fortran order, so the order is different from + // the cblas convention. + int lda = (transA == CblasNoTrans) ? K : M; + int ldb = (transB == CblasNoTrans) ? N : K; + int ldc = N; + cublasOperation_t cuTransA = + (transA == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; + cublasOperation_t cuTransB = + (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; + const int64_t strideC = M * N; + + CUBlas::GEMM_BATCH(context_.cublas_handle(), cuTransB, cuTransA, N, M, K, + &alpha, B, ldb, strideB, A, lda, strideA, &beta, C, ldc, + strideC, batchCount); +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/fluid/operators/math/blas_impl.h b/paddle/fluid/operators/math/blas_impl.h index 9db53ccfd..7360cc0a9 100644 --- a/paddle/fluid/operators/math/blas_impl.h +++ b/paddle/fluid/operators/math/blas_impl.h @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #pragma once - +#include #include "paddle/fluid/operators/math/math_function.h" namespace paddle { @@ -28,6 +28,23 @@ struct CBlas { static void GEMM(ARGS... args) { cblas_sgemm(args...); } + + template + static void AXPY(ARGS... args) { + cblas_saxpy(args...); + } + + template + static void GEMV(ARGS... args) { + cblas_sgemv(args...); + } + +#ifdef PADDLE_WITH_MKLML + template + static void GEMM_BATCH(ARGS... args) { + cblas_sgemm_batch(args...); + } +#endif }; template <> @@ -36,11 +53,33 @@ struct CBlas { static void GEMM(ARGS... args) { cblas_dgemm(args...); } + + template + static void AXPY(ARGS... args) { + cblas_daxpy(args...); + } + + template + static void GEMV(ARGS... args) { + cblas_dgemv(args...); + } + +#ifdef PADDLE_WITH_MKLML + template + static void GEMM_BATCH(ARGS... args) { + cblas_dgemm_batch(args...); + } +#endif }; template <> struct CBlas { static void GEMM(...) { PADDLE_THROW("float16 GEMM not supported on CPU"); } +#ifdef PADDLE_WITH_MKLML + static void GEMM_BATCH(...) { + PADDLE_THROW("float16 GEMM_BATCH not supported on CPU"); + } +#endif }; template <> @@ -93,6 +132,54 @@ void Blas::MatMul(const framework::Tensor &mat_a, bool trans_a, beta, mat_out->data()); } +template <> +template +void Blas::AXPY(int n, T alpha, const T *x, + T *y) const { + CBlas::AXPY(n, alpha, x, 1, y, 1); +} + +template <> +template +void Blas::GEMV(bool trans_a, int M, int N, T alpha, + const T *A, const T *B, T beta, + T *C) const { + CBLAS_TRANSPOSE transA = !trans_a ? CblasNoTrans : CblasTrans; + CBlas::GEMV(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); +} + +template <> +template +void Blas::BatchedGEMM( + CBLAS_TRANSPOSE transA, CBLAS_TRANSPOSE transB, int M, int N, int K, + T alpha, const T *A, const T *B, T beta, T *C, int batchCount, + int64_t strideA, int64_t strideB) const { +#ifdef PADDLE_WITH_MKLML + int lda = (transA == CblasNoTrans) ? K : M; + int ldb = (transB == CblasNoTrans) ? N : K; + int ldc = N; + auto a_array = std::vector(batchCount); + auto b_array = std::vector(batchCount); + auto c_array = std::vector(batchCount); + for (int k = 0; k < batchCount; ++k) { + a_array[k] = &A[k * strideA]; + b_array[k] = &B[k * strideB]; + c_array[k] = &C[k * M * N]; + } + + CBlas::GEMM_BATCH(CblasRowMajor, &transA, &transB, &M, &N, &K, &alpha, + a_array.data(), &lda, b_array.data(), &ldb, &beta, + c_array.data(), &ldc, 1 /* group_count */, &batchCount); +#else + for (int k = 0; k < batchCount; ++k) { + const float *Ak = &A[k * strideA]; + const float *Bk = &B[k * strideB]; + float *Ck = &C[k * M * N]; + this->template GEMM(transA, transB, M, N, K, alpha, Ak, Bk, beta, Ck); + } +#endif +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/fluid/operators/math/context_project.h b/paddle/fluid/operators/math/context_project.h index 027a019a2..bc0df3f35 100644 --- a/paddle/fluid/operators/math/context_project.h +++ b/paddle/fluid/operators/math/context_project.h @@ -17,8 +17,8 @@ limitations under the License. */ #include #include #include "paddle/fluid/framework/lod_tensor.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/im2col.h" -#include "paddle/fluid/operators/math/math_function.h" namespace paddle { namespace operators { @@ -211,6 +211,7 @@ class ContextProjectGradFunctor { int input_row_begin, input_row_end; int sequence_height, sequence_width; sequence_width = in.dims()[1]; + auto blas = math::GetBlas(context); if (input_grad) { for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { @@ -262,8 +263,8 @@ class ContextProjectGradFunctor { Tensor out_t_sub = out_t.Slice(k * context_length, k * context_length + padding_size); Tensor w_sub = padding_data->Slice(k, k + padding_size); - axpy(context, w_sub.numel(), static_cast(1), - out_t_sub.data(), w_sub.data()); + blas.AXPY(w_sub.numel(), static_cast(1), out_t_sub.data(), + w_sub.data()); } } if (down_pad > 0) { @@ -294,8 +295,8 @@ class ContextProjectGradFunctor { (down_pad_begin_row + t) * context_length); Tensor w_sub = padding_data->Slice( up_pad + padding_idx, up_pad + padding_idx + padding_size); - axpy(context, w_sub.numel(), static_cast(1), - out_t_sub.data(), w_sub.data()); + blas.AXPY(w_sub.numel(), static_cast(1), out_t_sub.data(), + w_sub.data()); } } out_t.Resize({sequence_height, context_length * sequence_width}); diff --git a/paddle/fluid/operators/math/gru_compute.cc b/paddle/fluid/operators/math/gru_compute.cc index d78625027..0e15b81de 100644 --- a/paddle/fluid/operators/math/gru_compute.cc +++ b/paddle/fluid/operators/math/gru_compute.cc @@ -10,9 +10,9 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/math/gru_compute.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/detail/gru_cpu_kernel.h" #include "paddle/fluid/operators/math/detail/gru_kernel.h" -#include "paddle/fluid/operators/math/math_function.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/math/gru_compute.cu b/paddle/fluid/operators/math/gru_compute.cu index f26bec410..1327d9149 100644 --- a/paddle/fluid/operators/math/gru_compute.cu +++ b/paddle/fluid/operators/math/gru_compute.cu @@ -10,10 +10,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #include +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/detail/gru_gpu_kernel.h" #include "paddle/fluid/operators/math/detail/gru_kernel.h" #include "paddle/fluid/operators/math/gru_compute.h" -#include "paddle/fluid/operators/math/math_function.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/math/math_function.cc b/paddle/fluid/operators/math/math_function.cc index f658a158d..d62ea387c 100644 --- a/paddle/fluid/operators/math/math_function.cc +++ b/paddle/fluid/operators/math/math_function.cc @@ -24,133 +24,6 @@ namespace math { using float16 = paddle::platform::float16; -template <> -void batched_gemm( - const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const float16 alpha, const float16* A, const float16* B, const float16 beta, - float16* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { - PADDLE_THROW("float16 batched_gemm not supported on CPU"); -} - -#ifdef PADDLE_WITH_MKLML -// Use cblas_{s,d}gemm_batched if available: Run with 1 group of size batchSize. -template <> -void batched_gemm( - const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const float alpha, const float* A, const float* B, const float beta, - float* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { - int lda = (transA == CblasNoTrans) ? K : M; - int ldb = (transB == CblasNoTrans) ? N : K; - int ldc = N; - auto a_array = std::vector(batchCount); - auto b_array = std::vector(batchCount); - auto c_array = std::vector(batchCount); - for (int k = 0; k < batchCount; ++k) { - a_array[k] = &A[k * strideA]; - b_array[k] = &B[k * strideB]; - c_array[k] = &C[k * M * N]; - } - cblas_sgemm_batch(CblasRowMajor, &transA, &transB, &M, &N, &K, &alpha, - a_array.data(), &lda, b_array.data(), &ldb, &beta, - c_array.data(), &ldc, 1 /* group_count */, &batchCount); -} - -template <> -void batched_gemm( - const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const double alpha, const double* A, const double* B, const double beta, - double* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { - int lda = (transA == CblasNoTrans) ? K : M; - int ldb = (transB == CblasNoTrans) ? N : K; - int ldc = N; - auto a_array = std::vector(batchCount); - auto b_array = std::vector(batchCount); - auto c_array = std::vector(batchCount); - for (int k = 0; k < batchCount; ++k) { - a_array[k] = &A[k * strideA]; - b_array[k] = &B[k * strideB]; - c_array[k] = &C[k * M * N]; - } - cblas_dgemm_batch(CblasRowMajor, &transA, &transB, &M, &N, &K, &alpha, - a_array.data(), &lda, b_array.data(), &ldb, &beta, - c_array.data(), &ldc, 1 /* group_count */, &batchCount); -} -#else -// The below is a naive but correct serial implementation that just loops -// over the batch dimension. This is a fallback for when the batched gemm -// functions of Intel MKL are not available. In the future, this computation -// should be parallelized. -template <> -void batched_gemm( - const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const float alpha, const float* A, const float* B, const float beta, - float* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { - for (int k = 0; k < batchCount; ++k) { - const float* Ak = &A[k * strideA]; - const float* Bk = &B[k * strideB]; - float* Ck = &C[k * M * N]; - Blas(context).GEMM(transA, transB, M, N, K, - alpha, Ak, Bk, beta, Ck); - } -} - -template <> -void batched_gemm( - const platform::CPUDeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const double alpha, const double* A, const double* B, const double beta, - double* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { - for (int k = 0; k < batchCount; ++k) { - const double* Ak = &A[k * strideA]; - const double* Bk = &B[k * strideB]; - double* Ck = &C[k * M * N]; - Blas(context).GEMM(transA, transB, M, N, K, - alpha, Ak, Bk, beta, Ck); - } -} -#endif - -template <> -void gemv( - const platform::CPUDeviceContext& context, const bool trans_a, const int M, - const int N, const float alpha, const float* A, const float* B, - const float beta, float* C) { - CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; - cblas_sgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); -} - -template <> -void gemv( - const platform::CPUDeviceContext& context, const bool trans_a, const int M, - const int N, const double alpha, const double* A, const double* B, - const double beta, double* C) { - CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; - cblas_dgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); -} - -template <> -void axpy( - const platform::CPUDeviceContext& context, const int n, const float alpha, - const float* x, float* y) { - cblas_saxpy(n, alpha, x, 1, y, 1); -} - -template <> -void axpy( - const platform::CPUDeviceContext& context, const int n, const double alpha, - const double* x, double* y) { - cblas_daxpy(n, alpha, x, 1, y, 1); -} - template struct SetConstant; template struct SetConstant; template struct SetConstant; diff --git a/paddle/fluid/operators/math/math_function.cu b/paddle/fluid/operators/math/math_function.cu index 15f7f0feb..b5bf84e51 100644 --- a/paddle/fluid/operators/math/math_function.cu +++ b/paddle/fluid/operators/math/math_function.cu @@ -15,6 +15,7 @@ limitations under the License. */ #define EIGEN_USE_GPU #include #include "paddle/fluid/framework/data_type.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/math_function_impl.h" #include "paddle/fluid/platform/float16.h" @@ -25,136 +26,6 @@ namespace math { using float16 = paddle::platform::float16; -template <> -void batched_gemm( - const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const float16 alpha, const float16* A, const float16* B, const float16 beta, - float16* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { -#if CUDA_VERSION >= 8000 - // Note that cublas follows fortran order, so the order is different from - // the cblas convention. - int lda = (transA == CblasNoTrans) ? K : M; - int ldb = (transB == CblasNoTrans) ? N : K; - int ldc = N; - cublasOperation_t cuTransA = - (transA == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - cublasOperation_t cuTransB = - (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - const int64_t strideC = M * N; - - const half h_alpha = static_cast(alpha); - const half h_beta = static_cast(beta); - const half* h_A = reinterpret_cast(A); - const half* h_B = reinterpret_cast(B); - half* h_C = reinterpret_cast(C); - - // TODO(kexinzhao): add processing code for compute capability < 53 case - PADDLE_ENFORCE_GE(context.GetComputeCapability(), 53, - "cublas Hgemm requires GPU compute capability >= 53"); - - PADDLE_ENFORCE(platform::dynload::cublasHgemmStridedBatched( - context.cublas_handle(), cuTransB, cuTransA, N, M, K, &h_alpha, h_B, ldb, - strideB, h_A, lda, strideA, &h_beta, h_C, ldc, strideC, batchCount)); -#else - PADDLE_ENFORCE(false, "HgemmStridedBatched is not supported on cuda <= 7.5"); -#endif -} - -template <> -void batched_gemm( - const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const float alpha, const float* A, const float* B, const float beta, - float* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { -#if CUDA_VERSION >= 8000 - // Note that cublas follows fortran order, so the order is different from - // the cblas convention. - int lda = (transA == CblasNoTrans) ? K : M; - int ldb = (transB == CblasNoTrans) ? N : K; - int ldc = N; - cublasOperation_t cuTransA = - (transA == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - cublasOperation_t cuTransB = - (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - const int64_t strideC = M * N; - - PADDLE_ENFORCE(platform::dynload::cublasSgemmStridedBatched( - context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, - strideB, A, lda, strideA, &beta, C, ldc, strideC, batchCount)); -#else - PADDLE_ENFORCE(false, "SgemmStridedBatched is not supported on cuda <= 7.5"); -#endif -} - -template <> -void batched_gemm( - const platform::CUDADeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, - const double alpha, const double* A, const double* B, const double beta, - double* C, const int batchCount, const int64_t strideA, - const int64_t strideB) { -#if CUDA_VERSION >= 8000 - // Note that cublas follows fortran order, so the order is different from - // the cblas convention. - int lda = (transA == CblasNoTrans) ? K : M; - int ldb = (transB == CblasNoTrans) ? N : K; - int ldc = N; - cublasOperation_t cuTransA = - (transA == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - cublasOperation_t cuTransB = - (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - const int64_t strideC = M * N; - - PADDLE_ENFORCE(platform::dynload::cublasDgemmStridedBatched( - context.cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, - strideB, A, lda, strideA, &beta, C, ldc, strideC, batchCount)); -#else - PADDLE_ENFORCE(false, "DgemmStridedBatched is not supported on cuda <= 7.5"); -#endif -} - -template <> -void gemv( - const platform::CUDADeviceContext& context, const bool trans_a, const int M, - const int N, const float alpha, const float* A, const float* B, - const float beta, float* C) { - cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; - - PADDLE_ENFORCE(platform::dynload::cublasSgemv(context.cublas_handle(), - cuTransA, N, M, &alpha, A, N, B, - 1, &beta, C, 1)); -} - -template <> -void gemv( - const platform::CUDADeviceContext& context, const bool trans_a, const int M, - const int N, const double alpha, const double* A, const double* B, - const double beta, double* C) { - cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; - PADDLE_ENFORCE(platform::dynload::cublasDgemv(context.cublas_handle(), - cuTransA, N, M, &alpha, A, N, B, - 1, &beta, C, 1)); -} - -template <> -void axpy( - const platform::CUDADeviceContext& context, const int n, const float alpha, - const float* x, float* y) { - PADDLE_ENFORCE(platform::dynload::cublasSaxpy(context.cublas_handle(), n, - &alpha, x, 1, y, 1)); -} - -template <> -void axpy( - const platform::CUDADeviceContext& context, const int n, const double alpha, - const double* x, double* y) { - PADDLE_ENFORCE(platform::dynload::cublasDaxpy(context.cublas_handle(), n, - &alpha, x, 1, y, 1)); -} - template struct SetConstant; template struct SetConstant; template struct SetConstant; @@ -246,10 +117,9 @@ void ColwiseSum::operator()( one.mutable_data({in_dims[0]}, context.GetPlace()); SetConstant set; set(context, &one, static_cast(1.0)); - gemv( - context, true, static_cast(in_dims[0]), static_cast(in_dims[1]), - 1.0, input.data(), one.data(), 0.0, - vector->data()); + GetBlas(context).GEMV( + true, static_cast(in_dims[0]), static_cast(in_dims[1]), 1.0, + input.data(), one.data(), 0.0, vector->data()); } template struct RowwiseSum; @@ -268,10 +138,9 @@ void RowwiseSum::operator()( one.mutable_data({size}, context.GetPlace()); SetConstant set; set(context, &one, static_cast(1.0)); - gemv( - context, true, static_cast(in_dims[1]), static_cast(in_dims[0]), - 1.0, one.data(), input.data(), 0.0, - vector->data()); + GetBlas(context).GEMV( + true, static_cast(in_dims[1]), static_cast(in_dims[0]), 1.0, + one.data(), input.data(), 0.0, vector->data()); } template struct RowwiseMean; diff --git a/paddle/fluid/operators/math/math_function.h b/paddle/fluid/operators/math/math_function.h index 589e41714..d4b0e17ed 100644 --- a/paddle/fluid/operators/math/math_function.h +++ b/paddle/fluid/operators/math/math_function.h @@ -51,94 +51,6 @@ int LAPACKE_dgetri(int matrix_layout, int n, double* a, int lda, namespace paddle { namespace operators { namespace math { - -// Support continuous memory now -// If transA = N, and transB = N -// Then matrixA: M * K, matrixB: K * N, matrixC : M * N -// For more detailed info, please refer to -// http://www.netlib.org/lapack/explore-html/d4/de2/sgemm_8f.html - -template -class Blas { - public: - explicit Blas(const DeviceContext& context) : context_(context) {} - - template - void GEMM(CBLAS_TRANSPOSE transA, CBLAS_TRANSPOSE transB, int M, int N, int K, - T alpha, const T* A, const T* B, T beta, T* C) const; - - template - void GEMM(bool transA, bool transB, int M, int N, int K, T alpha, const T* A, - int lda, const T* B, int ldb, T beta, T* C, int ldc) const; - - template - void MatMul(const framework::Tensor& mat_a, bool trans_a, - const framework::Tensor& mat_b, bool trans_b, T alpha, - framework::Tensor* mat_out, T beta) const; - - template - void MatMul(const framework::Tensor& mat_a, bool trans_a, - const framework::Tensor& mat_b, bool trans_b, - framework::Tensor* mat_out) const { - MatMul(mat_a, trans_a, mat_b, trans_b, static_cast(1.0), mat_out, - static_cast(0.0)); - } - - template - void MatMul(const framework::Tensor& mat_a, const framework::Tensor& mat_b, - framework::Tensor* mat_out) const { - this->template MatMul(mat_a, false, mat_b, false, mat_out); - } - - private: - const DeviceContext& context_; -}; - -template -class BlasT : private Blas { - public: - using Blas::Blas; - - template - void GEMM(ARGS... args) const { - static_cast*>(this)->template GEMM(args...); - } - - template - void MatMul(ARGS... args) const { - static_cast*>(this)->template MatMul(args...); - } -}; - -template -inline BlasT GetBlas( - const framework::ExecutionContext& exe_ctx) { - return BlasT( - exe_ctx.template device_context()); -} - -template -inline BlasT GetBlas(const DeviceContext& dev_ctx) { - return BlasT(dev_ctx); -} - -// Batched gemm -template -void batched_gemm(const DeviceContext& context, const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE transB, const int M, const int N, - const int K, const T alpha, const T* A, const T* B, - const T beta, T* C, const int batchCount, - const int64_t strideA, const int64_t strideB); - -template -void gemv(const DeviceContext& context, const bool trans_a, const int M, - const int N, const T alpha, const T* A, const T* B, const T beta, - T* C); - -template -void axpy(const DeviceContext& context, const int n, const T alpha, const T* x, - T* y); - template struct Transpose { void operator()(const DeviceContext& context, const framework::Tensor& in, @@ -185,8 +97,3 @@ struct RowwiseMean { } // namespace math } // namespace operators } // namespace paddle - -#include "paddle/fluid/operators/math/blas_impl.h" -#ifdef PADDLE_WITH_CUDA -#include "paddle/fluid/operators/math/blas_impl.cu.h" -#endif diff --git a/paddle/fluid/operators/math/math_function_test.cc b/paddle/fluid/operators/math/math_function_test.cc index 6d11dc8c7..3719a264e 100644 --- a/paddle/fluid/operators/math/math_function_test.cc +++ b/paddle/fluid/operators/math/math_function_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/fluid/operators/math/math_function.h" #include "gtest/gtest.h" +#include "paddle/fluid/operators/math/blas.h" template inline paddle::operators::math::BlasT @@ -129,9 +130,8 @@ void GemvTest(int m, int n, bool trans) { } paddle::platform::CPUDeviceContext context(*cpu_place); - paddle::operators::math::gemv( - context, trans, static_cast(m), static_cast(n), 1., data_a, - data_b, 0., data_c); + GetBlas(context).GEMV(trans, static_cast(m), static_cast(n), 1., + data_a, data_b, 0., data_c); if (!trans) { for (int i = 0; i < m; ++i) { diff --git a/paddle/fluid/operators/math/math_function_test.cu b/paddle/fluid/operators/math/math_function_test.cu index a6426120d..bcbb4a827 100644 --- a/paddle/fluid/operators/math/math_function_test.cu +++ b/paddle/fluid/operators/math/math_function_test.cu @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "gtest/gtest.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/platform/device_context.h" @@ -434,9 +435,8 @@ void GemvTest(int m, int n, bool trans) { paddle::framework::TensorCopySync(mat_a, gpu_place, &g_mat_a); paddle::framework::TensorCopySync(vec_b, gpu_place, &g_vec_b); - paddle::operators::math::gemv( - context, trans, static_cast(m), static_cast(n), 1., g_data_a, - g_data_b, 0., g_data_c); + GetBlas(context).GEMV(trans, static_cast(m), static_cast(n), 1., + g_data_a, g_data_b, 0., g_data_c); paddle::framework::TensorCopySync(g_vec_c, cpu_place, &vec_c); diff --git a/paddle/fluid/operators/math/matmul.h b/paddle/fluid/operators/math/matmul.h index 67efd1be5..87fd38a32 100644 --- a/paddle/fluid/operators/math/matmul.h +++ b/paddle/fluid/operators/math/matmul.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once #include #include -#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/operators/math/blas.h" namespace paddle { namespace operators { @@ -129,16 +129,17 @@ class MatMulFunctor { CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; + auto blas = GetBlas(context); + if (!batchCount) { // regular matrix multiplication - Blas(context).GEMM(transA, transB, M, N, kA, alpha, - a.data(), b.data(), beta, - out->data()); + blas.GEMM(transA, transB, M, N, kA, alpha, a.data(), b.data(), beta, + out->data()); } else { // batched matrix multiplication - batched_gemm( - context, transA, transB, M, N, kA, alpha, a.data(), b.data(), - beta, out->data(), batchCount, strideA, strideB); + blas.BatchedGEMM(transA, transB, M, N, kA, alpha, a.data(), + b.data(), beta, out->data(), batchCount, strideA, + strideB); } } }; diff --git a/paddle/fluid/operators/mul_op.h b/paddle/fluid/operators/mul_op.h index 776b3a7d4..15dd975e3 100644 --- a/paddle/fluid/operators/mul_op.h +++ b/paddle/fluid/operators/mul_op.h @@ -14,9 +14,9 @@ limitations under the License. */ #pragma once -#include "paddle/fluid/operators/math/math_function.h" - #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/blas.h" +#include "paddle/fluid/operators/math/math_function.h" namespace paddle { namespace operators { -- GitLab