From d330cf66cc8555d08ae6e3e4e83bf2a4096b968c Mon Sep 17 00:00:00 2001 From: Jack Zhou Date: Fri, 16 Oct 2020 10:55:23 +0800 Subject: [PATCH] Fix xpu enforce (#27978) * test=kunlun; Add elementwise XPU OP kernel for KUNLUN core, including (but still cannot process common broadcast): * elementwise_div op * elementwise_max op * elementwise_mul op (with grad op) * elementwise_sub op (with grad op) * 0.05->0.01 * add xpu error message description;test=kunlun --- .../elementwise/elementwise_add_op_xpu.cc | 8 ++- .../operators/elementwise/elementwise_xpu.h | 62 +++++++++++++------ .../fluid/tests/unittests/xpu/elementwise.py | 2 +- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc index 9ff7a71d7f..57a42d2615 100644 --- a/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc +++ b/paddle/fluid/operators/elementwise/elementwise_add_op_xpu.cc @@ -49,7 +49,8 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { int axis = ctx.Attr("axis"); PADDLE_ENFORCE_GE(dx_dims.size(), dy_dims_untrimed.size(), - "Rank of first input must >= rank of second input."); + platform::errors::InvalidArgument( + "Rank of first input must >= rank of second input.")); if (dx != nullptr) { dx->mutable_data(ctx.GetPlace()); @@ -69,8 +70,9 @@ class ElementwiseAddGradXPUKernel : public ElemwiseGradKernel { n = dout->numel(); } else { axis = (axis == -1 ? dx_dims.size() - dy_dims_untrimed.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < dx_dims.size(), - "Axis should be in range [0, dx_dims)"); + PADDLE_ENFORCE_EQ(axis >= 0 && axis < dx_dims.size(), true, + platform::errors::InvalidArgument( + "Axis should be in range [0, dx_dims)")); auto dy_dims = trim_trailing_singular_dims(dy_dims_untrimed); axis = (dy_dims.size() == 0) ? dx_dims.size() : axis; get_mid_dims(dx_dims, dy_dims, axis, &pre, &n, &post, diff --git a/paddle/fluid/operators/elementwise/elementwise_xpu.h b/paddle/fluid/operators/elementwise/elementwise_xpu.h index 9145ab856d..53f2cd2dcc 100644 --- a/paddle/fluid/operators/elementwise/elementwise_xpu.h +++ b/paddle/fluid/operators/elementwise/elementwise_xpu.h @@ -14,9 +14,25 @@ limitations under the License. */ #pragma once #ifdef PADDLE_WITH_XPU #include +#include #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/platform/place.h" +inline std::string get_xpu_error_message(int error_type) { + static std::unordered_map xpu_error_map = { + {baidu::xpu::api::INVALID_PARAM, "Parameter is invalid."}, + {baidu::xpu::api::RUNTIME_ERROR, + "Please check whether Baidu Kunlun Card " + "is properly installed."}, + {baidu::xpu::api::NO_ENOUGH_WORKSPACE, + "There is not enough memory in Baidu" + " Kunlun Card."}}; + if (xpu_error_map.find(error_type) == xpu_error_map.end()) { + return "Unknown error type!"; + } + return xpu_error_map[error_type]; +} + #define XPU_MALLOC(addr, num_bytes) \ PADDLE_ENFORCE_EQ(xpu_malloc(reinterpret_cast(addr), num_bytes), \ XPU_SUCCESS, \ @@ -102,21 +118,27 @@ limitations under the License. */ int res = \ xpu::broadcast_ew(dev_ctx.x_context(), y_data, y_broadcast, pre, \ n, post, xpu::ElementwiseOp::ASSIGN); \ - PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, \ - platform::errors::Fatal("XPU kernel error!")); \ + PADDLE_ENFORCE_EQ( \ + res, xpu::Error_t::SUCCESS, \ + platform::errors::External("XPU kernel error occur! %s", \ + get_xpu_error_message(res))); \ y_data = y_broadcast; \ } \ } \ int res = xpu::elementwise_##kernel_name##_grad( \ dev_ctx.x_context(), x_data, y_data, dout->data() /*out*/, \ dout->data(), dx_data, dy_data, len); \ - PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, \ - platform::errors::Fatal("XPU kernel error!")); \ + PADDLE_ENFORCE_EQ( \ + res, xpu::Error_t::SUCCESS, \ + platform::errors::External("XPU kernel error occur! %s", \ + get_xpu_error_message(res))); \ if ((dy != nullptr) && (len != n)) { \ int res = xpu::reduce_ew(dev_ctx.x_context(), dy_data, dy->data(), \ pre, n, post, xpu::ElementwiseOp::ASSIGN); \ - PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, \ - platform::errors::Fatal("XPU kernel error!")); \ + PADDLE_ENFORCE_EQ( \ + res, xpu::Error_t::SUCCESS, \ + platform::errors::External("XPU kernel error occur! %s", \ + get_xpu_error_message(res))); \ dev_ctx.Wait(); \ xpu_free(dy_data); \ } \ @@ -161,8 +183,8 @@ void XPUElementwise(const framework::ExecutionContext& ctx) { platform::errors::PreconditionNotMet( "This kernel only runs on XPU device.")); auto x_var = ctx.InputVar("X"); - PADDLE_ENFORCE_NE(x_var, nullptr, - platform::errors::Fatal("Cannot get input Variable X")); + PADDLE_ENFORCE_NE(x_var, nullptr, platform::errors::InvalidArgument( + "Cannot get input Variable X")); PADDLE_ENFORCE_EQ( x_var->IsType(), true, platform::errors::InvalidArgument( @@ -206,36 +228,36 @@ void XPUElementwise(const framework::ExecutionContext& ctx) { if (std::is_same>::value) { int res = xpu::matrix_vector_add(dev_ctx.x_context(), x_data, y_data, z_data, pre, n); - PADDLE_ENFORCE_EQ( - res, xpu::Error_t::SUCCESS, - platform::errors::Fatal("XPU kernel error! res = %d", res)); + PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, + platform::errors::External("XPU kernel error occur! %s", + get_xpu_error_message(res))); return; } if (std::is_same>::value) { int res = xpu::matrix_vector_mul(dev_ctx.x_context(), x_data, y_data, z_data, pre, n); - PADDLE_ENFORCE_EQ( - res, xpu::Error_t::SUCCESS, - platform::errors::Fatal("XPU kernel error! res = %d", res)); + PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, + platform::errors::External("XPU kernel error occur! %s", + get_xpu_error_message(res))); return; } } if (pre != 1 || post != 1) { - PADDLE_ENFORCE(xpu_malloc(reinterpret_cast(&y_broadcast), - len * sizeof(T)) == XPU_SUCCESS); + XPU_MALLOC(&y_broadcast, len * sizeof(T)); int res = xpu::broadcast_ew(dev_ctx.x_context(), y_data, y_broadcast, pre, n, post, xpu::ElementwiseOp::ASSIGN); - PADDLE_ENFORCE_EQ( - res, xpu::Error_t::SUCCESS, - platform::errors::Fatal("XPU kernel error! res = %d", res)); + PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, + platform::errors::External("XPU kernel error occur! %s", + get_xpu_error_message(res))); y_data = y_broadcast; } Functor functor; int res = functor(dev_ctx.x_context(), x_data, y_data, z_data, len); PADDLE_ENFORCE_EQ(res, xpu::Error_t::SUCCESS, - platform::errors::Fatal("XPU kernel error! res = %d", res)); + platform::errors::External("XPU kernel error occur! %s", + get_xpu_error_message(res))); if (pre != 1 || post != 1) { dev_ctx.Wait(); diff --git a/python/paddle/fluid/tests/unittests/xpu/elementwise.py b/python/paddle/fluid/tests/unittests/xpu/elementwise.py index 64a7abb54a..f4f2ddb19c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/elementwise.py +++ b/python/paddle/fluid/tests/unittests/xpu/elementwise.py @@ -54,7 +54,7 @@ class TestXPUElementwiseOpBase(object): inputs_to_check, output_names, no_grad_set=None, - max_relative_error=0.05): + max_relative_error=0.01): if self.grad_implemented and not self.is_common_broadcast \ and not self.is_x_size_less_than_y: if paddle.is_compiled_with_xpu(): -- GitLab