提交 d623e863 编写于 作者: A Adam 提交者: Tao Luo

Fix GELU grad error (#21204)

test=develop
上级 a152315b
...@@ -340,10 +340,8 @@ struct GeluFunctor : public BaseActivationFunctor<T> { ...@@ -340,10 +340,8 @@ struct GeluFunctor : public BaseActivationFunctor<T> {
} }
}; };
// gelu_grad(x) = dout * (0.5 * (1 + erf(x / sqrt(2))) + 0.5 * 2 / sqrt(pie) / // gelu_grad(x) = dout * (0.5 * (1 + erf(x / sqrt(2))) + 0.5 * 2 / sqrt(pi) /
// sqrt(2) * x * exp (-0.5 * sqrt(x))) // sqrt(2) * x * exp (-0.5 * x^2))
// gelu_grad(x) = dout * (0.5 + 0.5 * erf(x * M_SQRT1_2) + (0.5 * M_2_SQRTPI *
// M_SQRT1_2) * x * exp (-0.5 * sqrt(x)))
template <typename T> template <typename T>
struct GeluGradFunctor : BaseActivationFunctor<T> { struct GeluGradFunctor : BaseActivationFunctor<T> {
template <typename Device, typename X, typename Out, typename dOut, template <typename Device, typename X, typename Out, typename dOut,
...@@ -353,42 +351,35 @@ struct GeluGradFunctor : BaseActivationFunctor<T> { ...@@ -353,42 +351,35 @@ struct GeluGradFunctor : BaseActivationFunctor<T> {
!defined(__OSX__) && !defined(PADDLE_WITH_CUDA) !defined(__OSX__) && !defined(PADDLE_WITH_CUDA)
auto x_data = x.data(); auto x_data = x.data();
auto dx_data = dx.data(); auto dx_data = dx.data();
auto dout_data = dout.data();
int n = std::min(x.size(), dx.size()); int n = std::min(x.size(), dx.size());
std::memset(dx_data, 0, n * sizeof(T)); auto first = static_cast<T*>(std::malloc(n * sizeof(T)));
std::memset(first, 0, n * sizeof(T));
// First(dx_data) = erf(x * M_SQRT1_2)
math::CBlas<T>::AXPY(n, static_cast<T>(M_SQRT1_2), x_data, 1, dx_data, 1);
math::CBlas<T>::VMERF(n, dx_data, dx_data, VML_LA);
// Second = 0.5 * M_2_SQRTPI * M_SQRT1_2 * x * exp (-0.5 * sqrt(x))
auto second = static_cast<T*>(std::malloc(n * sizeof(T))); auto second = static_cast<T*>(std::malloc(n * sizeof(T)));
std::memset(second, 0, n * sizeof(T)); std::memset(second, 0, n * sizeof(T));
math::CBlas<T>::VSQUARE(n, x_data, second); // first = (0.5 * (1 + erf(x / sqrt(2))))
math::CBlas<T>::AXPY(n, static_cast<T>(M_SQRT1_2), x_data, 1, first, 1);
math::CBlas<T>::VMERF(n, first, first, VML_LA);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
second[i] *= static_cast<T>(-0.5); first[i] += static_cast<T>(1);
} }
math::CBlas<T>::SCAL(n, static_cast<T>(0.5), first, 1);
// second = (0.5 * 2/sqrt(pi) * 1/sqrt(2) * x * exp(-0.5 * x^2))
math::CBlas<T>::VSQUARE(n, x_data, second);
math::CBlas<T>::SCAL(n, -static_cast<T>(0.5), second, 1);
math::CBlas<T>::VEXP(n, second, second); math::CBlas<T>::VEXP(n, second, second);
math::CBlas<T>::VMUL(n, x_data, second, second); math::CBlas<T>::VMUL(n, x_data, second, second);
T tmp = static_cast<T>(0.5) * static_cast<T>(M_SQRT1_2) * math::CBlas<T>::SCAL(n, static_cast<T>(0.5 * M_2_SQRTPI * M_SQRT1_2),
static_cast<T>(M_2_SQRTPI); second, 1);
for (int i = 0; i < n; i++) {
second[i] *= tmp;
}
// Sum = 0.5 * First + Second
math::CBlas<T>::AXPY(n, static_cast<T>(0.5), dx_data, 1, second, 1);
// 0.5 + Sum
for (int i = 0; i < n; i++) {
second[i] += static_cast<T>(0.5);
}
// * dout // dx = dout * (first + second);
auto dout_data = dout.data(); math::CBlas<T>::VADD(n, first, second, first);
math::CBlas<T>::VMUL(n, dout_data, second, dx_data); math::CBlas<T>::VMUL(n, dout_data, first, dx_data);
std::free(first);
std::free(second); std::free(second);
#else #else
auto first = static_cast<T>(0.5) * auto first = static_cast<T>(0.5) *
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册