diff --git a/paddle/fluid/framework/data_type.h b/paddle/fluid/framework/data_type.h index 7aa7b7b2d96cf93dd2c0e3ba34b14307c230fa81..c8f73a5469ab32a5734d980010a52a6f72eb6ca8 100644 --- a/paddle/fluid/framework/data_type.h +++ b/paddle/fluid/framework/data_type.h @@ -20,6 +20,7 @@ limitations under the License. */ #include "paddle/fluid/platform/bfloat16.h" #include "paddle/fluid/platform/complex128.h" #include "paddle/fluid/platform/complex64.h" +#include "paddle/fluid/platform/eigen_ext.h" #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/float16.h" diff --git a/paddle/fluid/platform/bfloat16.h b/paddle/fluid/platform/bfloat16.h index f373e5ddb6d8ca9b38068b84bc99c6e6b5745a45..d1257f853e0e0e2b1541f541731408492ab8e7cf 100644 --- a/paddle/fluid/platform/bfloat16.h +++ b/paddle/fluid/platform/bfloat16.h @@ -15,22 +15,26 @@ #pragma once #include + +#include +#include #include + #if !defined(_WIN32) #define PADDLE_ALIGN(x) __attribute__((aligned(x))) #else #define PADDLE_ALIGN(x) __declspec(align(x)) #endif -#include - -#include "paddle/fluid/platform/hostdevice.h" -#include "unsupported/Eigen/CXX11/Tensor" - -namespace Eigen { -template -struct NumTraits; -} // namespace Eigen +#if (defined(__CUDACC__) || defined(__HIPCC__)) +#define HOSTDEVICE __host__ __device__ +#define DEVICE __device__ +#define HOST __host__ +#else +#define HOSTDEVICE +#define DEVICE +#define HOST +#endif namespace paddle { namespace platform { @@ -351,105 +355,3 @@ struct numeric_limits { }; } // namespace std - -namespace Eigen { - -using bfloat16 = paddle::platform::bfloat16; - -template <> -struct NumTraits : GenericNumTraits { - enum { - IsSigned = true, - IsInteger = false, - IsComplex = false, - RequireInitialization = false - }; - - HOSTDEVICE static inline bfloat16 epsilon() { - return paddle::platform::raw_uint16_to_bfloat16(0x3400); - } - HOSTDEVICE static inline bfloat16 dummy_precision() { - return bfloat16(1e-5f); - } - HOSTDEVICE static inline bfloat16 highest() { - return paddle::platform::raw_uint16_to_bfloat16(0x7f7f); - } - HOSTDEVICE static inline bfloat16 lowest() { - return paddle::platform::raw_uint16_to_bfloat16(0xff7f); - } - HOSTDEVICE static inline bfloat16 infinity() { - return paddle::platform::raw_uint16_to_bfloat16(0x7f80); - } - HOSTDEVICE static inline bfloat16 quiet_NaN() { - return paddle::platform::raw_uint16_to_bfloat16(0xffc1); - } -}; -namespace numext { - -template <> -HOSTDEVICE inline bool(isnan)(const bfloat16& a) { - return (paddle::platform::isnan)(a); -} - -template <> -HOSTDEVICE inline bool(isinf)(const bfloat16& a) { - return (paddle::platform::isinf)(a); -} - -template <> -HOSTDEVICE inline bool(isfinite)(const bfloat16& a) { - return (paddle::platform::isfinite)(a); -} - -template <> -HOSTDEVICE inline bfloat16 exp(const bfloat16& a) { - return bfloat16(::expf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 erf(const bfloat16& a) { - return bfloat16(::erff(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 log(const bfloat16& a) { - return bfloat16(::logf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 tanh(const bfloat16& a) { - return bfloat16(::tanhf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 sqrt(const bfloat16& a) { - return bfloat16(::sqrtf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 ceil(const bfloat16& a) { - return bfloat16(::ceilf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 floor(const bfloat16& a) { - return bfloat16(::floorf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 round(const bfloat16& a) { - return bfloat16(::roundf(static_cast(a))); -} - -template <> -HOSTDEVICE inline bfloat16 pow(const bfloat16& a, const bfloat16& b) { - return bfloat16(::powf(static_cast(a), static_cast(b))); -} - -template <> -HOSTDEVICE inline bfloat16 abs(const bfloat16& a) { - return bfloat16(::fabs(static_cast(a))); -} - -} // namespace numext -} // namespace Eigen diff --git a/paddle/fluid/platform/bfloat16_test.cc b/paddle/fluid/platform/bfloat16_test.cc index 3adfcd89be917862151c1a2b71890b59ebf38fbb..dc2d3aa73ba601362b071cf5aa23edd97cbc70b0 100644 --- a/paddle/fluid/platform/bfloat16_test.cc +++ b/paddle/fluid/platform/bfloat16_test.cc @@ -10,6 +10,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/platform/bfloat16.h" +#include "paddle/fluid/platform/eigen_ext.h" #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h #include "gtest/gtest.h" diff --git a/paddle/fluid/platform/complex128.h b/paddle/fluid/platform/complex128.h index c50ff2f810393db337380c44bb7da6a09017aefb..d6fddd672a0f890b5cae6a2c4c453f9da761da68 100644 --- a/paddle/fluid/platform/complex128.h +++ b/paddle/fluid/platform/complex128.h @@ -16,12 +16,10 @@ #include +#include +#include +#include #include -#if !defined(_WIN32) -#define PADDLE_ALIGN(x) __attribute__((aligned(x))) -#else -#define PADDLE_ALIGN(x) __declspec(align(x)) -#endif #ifdef PADDLE_WITH_CUDA #include @@ -33,15 +31,21 @@ #include // NOLINT #endif -#include - -#include "paddle/fluid/platform/hostdevice.h" -#include "unsupported/Eigen/CXX11/Tensor" +#if !defined(_WIN32) +#define PADDLE_ALIGN(x) __attribute__((aligned(x))) +#else +#define PADDLE_ALIGN(x) __declspec(align(x)) +#endif -namespace Eigen { -template -struct NumTraits; -} // namespace Eigen +#if (defined(__CUDACC__) || defined(__HIPCC__)) +#define HOSTDEVICE __host__ __device__ +#define DEVICE __device__ +#define HOST __host__ +#else +#define HOSTDEVICE +#define DEVICE +#define HOST +#endif namespace paddle { namespace platform { @@ -509,97 +513,5 @@ struct numeric_limits { }; } // namespace std -namespace Eigen { - -using complex128 = paddle::platform::complex128; - -template <> -struct NumTraits : GenericNumTraits> { - typedef double Real; - typedef typename NumTraits::Literal Literal; - enum { - IsComplex = 1, - RequireInitialization = NumTraits::RequireInitialization, - ReadCost = 2 * NumTraits::ReadCost, - AddCost = 2 * NumTraits::AddCost, - MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost - }; - - EIGEN_DEVICE_FUNC - static inline Real epsilon() { return NumTraits::epsilon(); } - EIGEN_DEVICE_FUNC - static inline Real dummy_precision() { - return NumTraits::dummy_precision(); - } - EIGEN_DEVICE_FUNC - static inline int digits10() { return NumTraits::digits10(); } -}; -namespace numext { - -template <> -HOSTDEVICE inline bool(isnan)(const complex128& a) { - return (paddle::platform::isnan)(a); -} - -template <> -HOSTDEVICE inline bool(isinf)(const complex128& a) { - return (paddle::platform::isinf)(a); -} - -template <> -HOSTDEVICE inline bool(isfinite)(const complex128& a) { - return (paddle::platform::isfinite)(a); -} - -template <> -HOSTDEVICE inline complex128 exp(const complex128& a) { - double com = ::expf(a.real); - double res_real = com * ::cosf(a.imag); - double res_imag = com * ::sinf(a.imag); - return complex128(res_real, res_imag); -} - -template <> -HOSTDEVICE inline complex128 log(const complex128& a) { - return paddle::platform::log(a); -} - -template <> -HOSTDEVICE inline complex128 tanh(const complex128& a) { - return paddle::platform::tanh(a); -} - -template <> -HOSTDEVICE inline complex128 sqrt(const complex128& a) { - return paddle::platform::sqrt(a); -} - -template <> -HOSTDEVICE inline complex128 ceil(const complex128& a) { - return complex128(::ceilf(a.real), ::ceilf(a.imag)); -} - -template <> -HOSTDEVICE inline complex128 floor(const complex128& a) { - return complex128(::floorf(a.real), ::floor(a.imag)); -} - -template <> -HOSTDEVICE inline complex128 round(const complex128& a) { - return complex128(::roundf(a.real), ::roundf(a.imag)); -} - -template <> -HOSTDEVICE inline complex128 pow(const complex128& a, const complex128& b) { - return paddle::platform::pow(a, b); -} - -template <> -HOSTDEVICE inline double abs(const complex128& a) { - return paddle::platform::abs(a); -} - -} // namespace numext -} // namespace Eigen #define MKL_Complex16 paddle::platform::complex128 diff --git a/paddle/fluid/platform/complex64.h b/paddle/fluid/platform/complex64.h index b91fdbab28b0b6d6e9529b3c31d4a118e97d3f11..9d55ba19105a617266f4aa77594c9da1b54145fe 100644 --- a/paddle/fluid/platform/complex64.h +++ b/paddle/fluid/platform/complex64.h @@ -15,12 +15,11 @@ #pragma once #include + +#include +#include +#include #include -#if !defined(_WIN32) -#define PADDLE_ALIGN(x) __attribute__((aligned(x))) -#else -#define PADDLE_ALIGN(x) __declspec(align(x)) -#endif #ifdef PADDLE_WITH_CUDA #include @@ -32,16 +31,23 @@ #include // NOLINT #endif -#include +#if !defined(_WIN32) +#define PADDLE_ALIGN(x) __attribute__((aligned(x))) +#else +#define PADDLE_ALIGN(x) __declspec(align(x)) +#endif -#include "paddle/fluid/platform/complex128.h" -#include "paddle/fluid/platform/hostdevice.h" -#include "unsupported/Eigen/CXX11/Tensor" +#if (defined(__CUDACC__) || defined(__HIPCC__)) +#define HOSTDEVICE __host__ __device__ +#define DEVICE __device__ +#define HOST __host__ +#else +#define HOSTDEVICE +#define DEVICE +#define HOST +#endif -namespace Eigen { -template -struct NumTraits; -} // namespace Eigen +#include "complex128.h" // NOLINT namespace paddle { namespace platform { @@ -510,98 +516,5 @@ struct numeric_limits { }; } // namespace std -namespace Eigen { - -using complex64 = paddle::platform::complex64; - -template <> -struct NumTraits : GenericNumTraits> { - typedef float Real; - typedef typename NumTraits::Literal Literal; - enum { - IsComplex = 1, - RequireInitialization = NumTraits::RequireInitialization, - ReadCost = 2 * NumTraits::ReadCost, - AddCost = 2 * NumTraits::AddCost, - MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost - }; - - EIGEN_DEVICE_FUNC - static inline Real epsilon() { return NumTraits::epsilon(); } - EIGEN_DEVICE_FUNC - static inline Real dummy_precision() { - return NumTraits::dummy_precision(); - } - EIGEN_DEVICE_FUNC - static inline int digits10() { return NumTraits::digits10(); } -}; - -namespace numext { - -template <> -HOSTDEVICE inline bool(isnan)(const complex64& a) { - return (paddle::platform::isnan)(a); -} - -template <> -HOSTDEVICE inline bool(isinf)(const complex64& a) { - return (paddle::platform::isinf)(a); -} - -template <> -HOSTDEVICE inline bool(isfinite)(const complex64& a) { - return (paddle::platform::isfinite)(a); -} - -template <> -HOSTDEVICE inline complex64 exp(const complex64& a) { - float com = ::expf(a.real); - float res_real = com * ::cosf(a.imag); - float res_imag = com * ::sinf(a.imag); - return complex64(res_real, res_imag); -} - -template <> -HOSTDEVICE inline complex64 log(const complex64& a) { - return paddle::platform::log(a); -} - -template <> -HOSTDEVICE inline complex64 tanh(const complex64& a) { - return paddle::platform::tanh(a); -} - -template <> -HOSTDEVICE inline complex64 sqrt(const complex64& a) { - return paddle::platform::sqrt(a); -} - -template <> -HOSTDEVICE inline complex64 ceil(const complex64& a) { - return complex64(::ceilf(a.real), ::ceilf(a.imag)); -} - -template <> -HOSTDEVICE inline complex64 floor(const complex64& a) { - return complex64(::floorf(a.real), ::floor(a.imag)); -} - -template <> -HOSTDEVICE inline complex64 round(const complex64& a) { - return complex64(::roundf(a.real), ::roundf(a.imag)); -} - -template <> -HOSTDEVICE inline complex64 pow(const complex64& a, const complex64& b) { - return paddle::platform::pow(a, b); -} - -template <> -HOSTDEVICE inline float abs(const complex64& a) { - return paddle::platform::abs(a); -} - -} // namespace numext -} // namespace Eigen #define MKL_Complex8 paddle::platform::complex64 diff --git a/paddle/fluid/platform/eigen_ext.h b/paddle/fluid/platform/eigen_ext.h new file mode 100644 index 0000000000000000000000000000000000000000..9e2c3630468e81382513e5b16951f1d2d03c3bcd --- /dev/null +++ b/paddle/fluid/platform/eigen_ext.h @@ -0,0 +1,306 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/platform/bfloat16.h" +#include "paddle/fluid/platform/complex128.h" +#include "paddle/fluid/platform/complex64.h" +#include "paddle/fluid/platform/hostdevice.h" + +#include "unsupported/Eigen/CXX11/Tensor" + +namespace Eigen { + +using bfloat16 = paddle::platform::bfloat16; +using complex64 = paddle::platform::complex64; +using complex128 = paddle::platform::complex128; + +template +struct NumTraits; + +template <> +struct NumTraits : GenericNumTraits { + enum { + IsSigned = true, + IsInteger = false, + IsComplex = false, + RequireInitialization = false + }; + + HOSTDEVICE static inline bfloat16 epsilon() { + return paddle::platform::raw_uint16_to_bfloat16(0x3400); + } + HOSTDEVICE static inline bfloat16 dummy_precision() { + return bfloat16(1e-5f); + } + HOSTDEVICE static inline bfloat16 highest() { + return paddle::platform::raw_uint16_to_bfloat16(0x7f7f); + } + HOSTDEVICE static inline bfloat16 lowest() { + return paddle::platform::raw_uint16_to_bfloat16(0xff7f); + } + HOSTDEVICE static inline bfloat16 infinity() { + return paddle::platform::raw_uint16_to_bfloat16(0x7f80); + } + HOSTDEVICE static inline bfloat16 quiet_NaN() { + return paddle::platform::raw_uint16_to_bfloat16(0xffc1); + } +}; + +template <> +struct NumTraits : GenericNumTraits> { + typedef float Real; + typedef typename NumTraits::Literal Literal; + enum { + IsComplex = 1, + RequireInitialization = NumTraits::RequireInitialization, + ReadCost = 2 * NumTraits::ReadCost, + AddCost = 2 * NumTraits::AddCost, + MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost + }; + + EIGEN_DEVICE_FUNC + static inline Real epsilon() { return NumTraits::epsilon(); } + EIGEN_DEVICE_FUNC + static inline Real dummy_precision() { + return NumTraits::dummy_precision(); + } + EIGEN_DEVICE_FUNC + static inline int digits10() { return NumTraits::digits10(); } +}; + +template <> +struct NumTraits : GenericNumTraits> { + typedef double Real; + typedef typename NumTraits::Literal Literal; + enum { + IsComplex = 1, + RequireInitialization = NumTraits::RequireInitialization, + ReadCost = 2 * NumTraits::ReadCost, + AddCost = 2 * NumTraits::AddCost, + MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost + }; + + EIGEN_DEVICE_FUNC + static inline Real epsilon() { return NumTraits::epsilon(); } + EIGEN_DEVICE_FUNC + static inline Real dummy_precision() { + return NumTraits::dummy_precision(); + } + EIGEN_DEVICE_FUNC + static inline int digits10() { return NumTraits::digits10(); } +}; + +namespace numext { + +//////////// bfloat methods ///////////// + +template <> +HOSTDEVICE inline bool(isnan)(const bfloat16& a) { + return (paddle::platform::isnan)(a); +} + +template <> +HOSTDEVICE inline bool(isinf)(const bfloat16& a) { + return (paddle::platform::isinf)(a); +} + +template <> +HOSTDEVICE inline bool(isfinite)(const bfloat16& a) { + return (paddle::platform::isfinite)(a); +} + +template <> +HOSTDEVICE inline bfloat16 exp(const bfloat16& a) { + return bfloat16(::expf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 erf(const bfloat16& a) { + return bfloat16(::erff(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 log(const bfloat16& a) { + return bfloat16(::logf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 tanh(const bfloat16& a) { + return bfloat16(::tanhf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 sqrt(const bfloat16& a) { + return bfloat16(::sqrtf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 ceil(const bfloat16& a) { + return bfloat16(::ceilf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 floor(const bfloat16& a) { + return bfloat16(::floorf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 round(const bfloat16& a) { + return bfloat16(::roundf(static_cast(a))); +} + +template <> +HOSTDEVICE inline bfloat16 pow(const bfloat16& a, const bfloat16& b) { + return bfloat16(::powf(static_cast(a), static_cast(b))); +} + +template <> +HOSTDEVICE inline bfloat16 abs(const bfloat16& a) { + return bfloat16(::fabs(static_cast(a))); +} + +//////////// complex64 methods ///////////// + +template <> +HOSTDEVICE inline bool(isnan)(const complex64& a) { + return (paddle::platform::isnan)(a); +} + +template <> +HOSTDEVICE inline bool(isinf)(const complex64& a) { + return (paddle::platform::isinf)(a); +} + +template <> +HOSTDEVICE inline bool(isfinite)(const complex64& a) { + return (paddle::platform::isfinite)(a); +} + +template <> +HOSTDEVICE inline complex64 exp(const complex64& a) { + float com = ::expf(a.real); + float res_real = com * ::cosf(a.imag); + float res_imag = com * ::sinf(a.imag); + return complex64(res_real, res_imag); +} + +template <> +HOSTDEVICE inline complex64 log(const complex64& a) { + return paddle::platform::log(a); +} + +template <> +HOSTDEVICE inline complex64 tanh(const complex64& a) { + return paddle::platform::tanh(a); +} + +template <> +HOSTDEVICE inline complex64 sqrt(const complex64& a) { + return paddle::platform::sqrt(a); +} + +template <> +HOSTDEVICE inline complex64 ceil(const complex64& a) { + return complex64(::ceilf(a.real), ::ceilf(a.imag)); +} + +template <> +HOSTDEVICE inline complex64 floor(const complex64& a) { + return complex64(::floorf(a.real), ::floor(a.imag)); +} + +template <> +HOSTDEVICE inline complex64 round(const complex64& a) { + return complex64(::roundf(a.real), ::roundf(a.imag)); +} + +template <> +HOSTDEVICE inline complex64 pow(const complex64& a, const complex64& b) { + return paddle::platform::pow(a, b); +} + +template <> +HOSTDEVICE inline float abs(const complex64& a) { + return paddle::platform::abs(a); +} + +//////////// complex128 methods ///////////// + +template <> +HOSTDEVICE inline bool(isnan)(const complex128& a) { + return (paddle::platform::isnan)(a); +} + +template <> +HOSTDEVICE inline bool(isinf)(const complex128& a) { + return (paddle::platform::isinf)(a); +} + +template <> +HOSTDEVICE inline bool(isfinite)(const complex128& a) { + return (paddle::platform::isfinite)(a); +} + +template <> +HOSTDEVICE inline complex128 exp(const complex128& a) { + double com = ::expf(a.real); + double res_real = com * ::cosf(a.imag); + double res_imag = com * ::sinf(a.imag); + return complex128(res_real, res_imag); +} + +template <> +HOSTDEVICE inline complex128 log(const complex128& a) { + return paddle::platform::log(a); +} + +template <> +HOSTDEVICE inline complex128 tanh(const complex128& a) { + return paddle::platform::tanh(a); +} + +template <> +HOSTDEVICE inline complex128 sqrt(const complex128& a) { + return paddle::platform::sqrt(a); +} + +template <> +HOSTDEVICE inline complex128 ceil(const complex128& a) { + return complex128(::ceilf(a.real), ::ceilf(a.imag)); +} + +template <> +HOSTDEVICE inline complex128 floor(const complex128& a) { + return complex128(::floorf(a.real), ::floor(a.imag)); +} + +template <> +HOSTDEVICE inline complex128 round(const complex128& a) { + return complex128(::roundf(a.real), ::roundf(a.imag)); +} + +template <> +HOSTDEVICE inline complex128 pow(const complex128& a, const complex128& b) { + return paddle::platform::pow(a, b); +} + +template <> +HOSTDEVICE inline double abs(const complex128& a) { + return paddle::platform::abs(a); +} + +} // namespace numext +} // namespace Eigen