Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 2d36c9a9 编写于 作者: G gouzil 提交者： GitHub
浏览文件
操作

【Hackathon No.70】[PHI decoupling] move jit kernels from fluid to phi (#50911) 

* [phi] move jit kernels from fluid to phi

* [phi] fix paddle::phi err

* [phi] fix windows 'posix_memalign': identifier not found

* [phi] fix windows 'posix_memalign_free': identifier not found

* [phi] fix readme directory structure, fc_functor  paddle::platform
上级 3b7c9ffc
隐藏空白更改
内联 并排
Showing with 653 addition and 744 deletion
paddle/fluid/operators/CMakeLists.txt
浏览文件 @ 2d36c9a9
......@@ -24,7 +24,6 @@ add_subdirectory(optimizers)
add_subdirectory(reduce_ops)
add_subdirectory(sequence_ops)
add_subdirectory(string)
add_subdirectory(jit)
add_subdirectory(prim_ops)
......
paddle/fluid/operators/crf_decoding_op.h
浏览文件 @ 2d36c9a9
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/fluid/framework/eigen.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace paddle {
......@@ -137,9 +137,9 @@ class CRFDecodingOpKernel : public framework::OpKernel<T> {
phi::DenseTensor track;
int* track_value =
track.mutable_data<int>(emission_dims, platform::CPUPlace());
auto ker =
jit::KernelFuncs<jit::CRFDecodingTuple<T>, platform::CPUPlace>::Cache()
.At(tag_num);
auto ker = phi::jit::KernelFuncs<phi::jit::CRFDecodingTuple<T>,
platform::CPUPlace>::Cache()
.At(tag_num);
ker(static_cast<int>(seq_len), x, w, alpha_value, track_value, tag_num);
T max_score = -std::numeric_limits<T>::max();
int max_i = 0;
......
paddle/fluid/operators/fused/fused_embedding_seq_pool_op.h
浏览文件 @ 2d36c9a9
......@@ -22,8 +22,8 @@ limitations under the License. */
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/framework/selected_rows_utils.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -108,17 +108,17 @@ struct EmbeddingVSumFunctor {
"But received the ids's LoD[0] = %d.",
ids_lod.size()));
jit::emb_seq_pool_attr_t attr(table_height,
table_width,
0,
idx_width,
out_width,
jit::SeqPoolType::kSum);
phi::jit::emb_seq_pool_attr_t attr(table_height,
table_width,
0,
idx_width,
out_width,
phi::jit::SeqPoolType::kSum);
for (size_t i = 0; i != ids_lod.size() - 1; ++i) {
attr.index_height = ids_lod[i + 1] - ids_lod[i];
auto emb_seqpool =
jit::KernelFuncs<jit::EmbSeqPoolTuple<T>, platform::CPUPlace>::Cache()
.At(attr);
auto emb_seqpool = phi::jit::KernelFuncs<phi::jit::EmbSeqPoolTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
emb_seqpool(
table, ids + ids_lod[i] * idx_width, output + i * out_width, &attr);
}
......@@ -265,9 +265,9 @@ class FusedEmbeddingSeqPoolGradKernel : public framework::OpKernel<T> {
T *d_table_data = d_table_value->mutable_data<T>(context.GetPlace());
const T *d_output_data = d_output->data<T>();
auto vbroadcast =
jit::KernelFuncs<jit::VBroadcastTuple<T>, platform::CPUPlace>::Cache()
.At(out_width);
auto vbroadcast = phi::jit::KernelFuncs<phi::jit::VBroadcastTuple<T>,
platform::CPUPlace>::Cache()
.At(out_width);
for (int i = 0; i < static_cast<int>(lod.size()) - 1; ++i) {
int64_t h = static_cast<int64_t>(lod[i + 1] - lod[i]);
const T *src = d_output_data + i * out_width;
......
paddle/fluid/operators/fused/fusion_gru_op.cc
浏览文件 @ 2d36c9a9
......@@ -19,9 +19,9 @@ limitations under the License. */
#include <vector>
#include "paddle/fluid/framework/op_version_registry.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/fc_functor.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/sequence2batch.h"
namespace paddle {
......@@ -273,33 +273,33 @@ class FusionGRUKernel : public framework::OpKernel<T> {
const int total_T = x_mat_dims[0]; \
const int D3 = wh_dims[1]
#define INIT_OTHER_DEFINES \
auto* h0 = ctx.Input<phi::DenseTensor>("H0"); \
auto* wx = ctx.Input<phi::DenseTensor>("WeightX"); \
auto* bias = ctx.Input<phi::DenseTensor>("Bias"); \
auto* hidden_out = ctx.Output<phi::DenseTensor>("Hidden"); \
bool is_reverse = ctx.Attr<bool>("is_reverse"); \
const int M = x_mat_dims[1]; \
const int D = wh_dims[0]; \
const int D2 = D * 2; \
const jit::gru_attr_t attr( \
D, \
jit::to_kerneltype(ctx.Attr<std::string>("gate_activation")), \
jit::to_kerneltype(ctx.Attr<std::string>("activation"))); \
jit::gru_t one_step; \
auto ComputeH1 = \
jit::KernelFuncs<jit::GRUH1Tuple<T>, platform::CPUPlace>::Cache().At( \
attr); \
auto ComputeHtPart1 = \
jit::KernelFuncs<jit::GRUHtPart1Tuple<T>, platform::CPUPlace>::Cache() \
.At(attr); \
auto ComputeHtPart2 = \
jit::KernelFuncs<jit::GRUHtPart2Tuple<T>, platform::CPUPlace>::Cache() \
.At(attr); \
const T* x_data = x->data<T>(); \
const T* wx_data = wx->data<T>(); \
const T* wh_data = wh->data<T>(); \
auto place = ctx.GetPlace(); \
#define INIT_OTHER_DEFINES \
auto* h0 = ctx.Input<phi::DenseTensor>("H0"); \
auto* wx = ctx.Input<phi::DenseTensor>("WeightX"); \
auto* bias = ctx.Input<phi::DenseTensor>("Bias"); \
auto* hidden_out = ctx.Output<phi::DenseTensor>("Hidden"); \
bool is_reverse = ctx.Attr<bool>("is_reverse"); \
const int M = x_mat_dims[1]; \
const int D = wh_dims[0]; \
const int D2 = D * 2; \
const phi::jit::gru_attr_t attr( \
D, \
phi::jit::to_kerneltype(ctx.Attr<std::string>("gate_activation")), \
phi::jit::to_kerneltype(ctx.Attr<std::string>("activation"))); \
phi::jit::gru_t one_step; \
auto ComputeH1 = phi::jit::KernelFuncs<phi::jit::GRUH1Tuple<T>, \
platform::CPUPlace>::Cache() \
.At(attr); \
auto ComputeHtPart1 = phi::jit::KernelFuncs<phi::jit::GRUHtPart1Tuple<T>, \
platform::CPUPlace>::Cache() \
.At(attr); \
auto ComputeHtPart2 = phi::jit::KernelFuncs<phi::jit::GRUHtPart2Tuple<T>, \
platform::CPUPlace>::Cache() \
.At(attr); \
const T* x_data = x->data<T>(); \
const T* wx_data = wx->data<T>(); \
const T* wh_data = wh->data<T>(); \
auto place = ctx.GetPlace(); \
T* xx_data = xx->mutable_data<T>(place)
void SeqCompute(const framework::ExecutionContext& ctx) const {
......
paddle/fluid/operators/fused/fusion_lstm_op.cc
浏览文件 @ 2d36c9a9
......@@ -16,9 +16,9 @@ limitations under the License. */
#include <string>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/fc_functor.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/sequence2batch.h"
namespace paddle {
......@@ -320,35 +320,35 @@ class FuisonLSTMKernel : public framework::OpKernel<T> {
const int D = wh_dims[0]; \
const int D4 = wh_dims[1]
#define INIT_OTHER_DEFINES \
const T* x_data = x->data<T>(); \
const T* wx_data = wx->data<T>(); \
const T* wh_data = wh->data<T>(); \
/* diagonal weight*/ \
const T* wp_data = bias->data<T>() + D4; \
/* for peephole only*/ \
T* checked_cell_data = nullptr; \
auto place = ctx.GetPlace(); \
if (use_peepholes) { \
/* w_ic * Ct-1, w_fc * Ct-1 ; w_oc * Ct => ih*/ \
auto* checked_cell = ctx.Output<phi::DenseTensor>("CheckedCell"); \
checked_cell_data = checked_cell->mutable_data<T>(place); \
} \
const jit::lstm_attr_t attr( \
D, \
jit::to_kerneltype(ctx.Attr<std::string>("gate_activation")), \
jit::to_kerneltype(ctx.Attr<std::string>("candidate_activation")), \
jit::to_kerneltype(ctx.Attr<std::string>("cell_activation")), \
use_peepholes); \
jit::lstm_t one_step; \
one_step.wp = wp_data; \
one_step.checked = checked_cell_data; \
auto ComputeC1H1 = \
jit::KernelFuncs<jit::LSTMC1H1Tuple<T>, platform::CPUPlace>::Cache().At( \
attr); \
auto ComputeCtHt = \
jit::KernelFuncs<jit::LSTMCtHtTuple<T>, platform::CPUPlace>::Cache().At( \
attr)
#define INIT_OTHER_DEFINES \
const T* x_data = x->data<T>(); \
const T* wx_data = wx->data<T>(); \
const T* wh_data = wh->data<T>(); \
/* diagonal weight*/ \
const T* wp_data = bias->data<T>() + D4; \
/* for peephole only*/ \
T* checked_cell_data = nullptr; \
auto place = ctx.GetPlace(); \
if (use_peepholes) { \
/* w_ic * Ct-1, w_fc * Ct-1 ; w_oc * Ct => ih*/ \
auto* checked_cell = ctx.Output<phi::DenseTensor>("CheckedCell"); \
checked_cell_data = checked_cell->mutable_data<T>(place); \
} \
const phi::jit::lstm_attr_t attr( \
D, \
phi::jit::to_kerneltype(ctx.Attr<std::string>("gate_activation")), \
phi::jit::to_kerneltype(ctx.Attr<std::string>("candidate_activation")), \
phi::jit::to_kerneltype(ctx.Attr<std::string>("cell_activation")), \
use_peepholes); \
phi::jit::lstm_t one_step; \
one_step.wp = wp_data; \
one_step.checked = checked_cell_data; \
auto ComputeC1H1 = phi::jit::KernelFuncs<phi::jit::LSTMC1H1Tuple<T>, \
platform::CPUPlace>::Cache() \
.At(attr); \
auto ComputeCtHt = phi::jit::KernelFuncs<phi::jit::LSTMCtHtTuple<T>, \
platform::CPUPlace>::Cache() \
.At(attr)
// Wh GEMM
#define GEMM_WH_ADDON(bs, prev, out) \
......
paddle/fluid/operators/fused/fusion_repeated_fc_relu_op.cc
浏览文件 @ 2d36c9a9
......@@ -17,7 +17,7 @@
#include <string>
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -122,14 +122,17 @@ void FusionRepeatedFCReluOpMaker::Make() {
}
template <typename T>
static void fc_relu(
const T* x, const T* w, const T* b, T* y, const jit::matmul_attr_t& attr) {
auto matmul =
jit::KernelFuncs<jit::MatMulTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto addbias_relu =
jit::KernelFuncs<jit::VAddReluTuple<T>, platform::CPUPlace>::Cache().At(
attr.n);
static void fc_relu(const T* x,
const T* w,
const T* b,
T* y,
const phi::jit::matmul_attr_t& attr) {
auto matmul = phi::jit::KernelFuncs<phi::jit::MatMulTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
auto addbias_relu = phi::jit::KernelFuncs<phi::jit::VAddReluTuple<T>,
platform::CPUPlace>::Cache()
.At(attr.n);
matmul(x, w, y, &attr);
T* dst = y;
for (int i = 0; i < attr.m; ++i) {
......@@ -152,7 +155,7 @@ class FusionRepeatedFCReluKernel : public framework::OpKernel<T> {
auto i_dims = in->dims();
const auto& w_dims = weights[0]->dims();
jit::matmul_attr_t attr;
phi::jit::matmul_attr_t attr;
attr.m = i_dims[0];
attr.n = w_dims[1];
attr.k = w_dims[0];
......
paddle/fluid/operators/fused/fusion_seqpool_concat_op.cc
浏览文件 @ 2d36c9a9
......@@ -17,7 +17,7 @@
#include <string>
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -121,15 +121,15 @@ class FusionSeqPoolConcatKernel : public framework::OpKernel<T> {
"dims[1] is %d, w is %d.",
y_dims[1],
w));
jit::seq_pool_attr_t attr(w, jit::SeqPoolType::kSum);
phi::jit::seq_pool_attr_t attr(w, phi::jit::SeqPoolType::kSum);
if (pooltype == "AVERAGE") {
attr.type = jit::SeqPoolType::kAvg;
attr.type = phi::jit::SeqPoolType::kAvg;
} else if (pooltype == "SQRT") {
attr.type = jit::SeqPoolType::kSqrt;
attr.type = phi::jit::SeqPoolType::kSqrt;
}
auto seqpool =
jit::KernelFuncs<jit::SeqPoolTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto seqpool = phi::jit::KernelFuncs<phi::jit::SeqPoolTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
size_t n = ins.size();
size_t dst_step_size = n * w;
for (size_t i = 0; i < n; ++i) {
......
paddle/fluid/operators/fused/fusion_seqpool_cvm_concat_op.cc
浏览文件 @ 2d36c9a9
......@@ -17,7 +17,7 @@
#include <string>
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -122,15 +122,15 @@ class FusionSeqPoolCVMConcatKernel : public framework::OpKernel<T> {
0,
paddle::platform::errors::InvalidArgument(
"The output of dims[1] should be dividable of w"));
jit::seq_pool_attr_t attr(w, jit::SeqPoolType::kSum);
phi::jit::seq_pool_attr_t attr(w, phi::jit::SeqPoolType::kSum);
if (pooltype == "AVERAGE") {
attr.type = jit::SeqPoolType::kAvg;
attr.type = phi::jit::SeqPoolType::kAvg;
} else if (pooltype == "SQRT") {
attr.type = jit::SeqPoolType::kSqrt;
attr.type = phi::jit::SeqPoolType::kSqrt;
}
auto seqpool =
jit::KernelFuncs<jit::SeqPoolTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto seqpool = phi::jit::KernelFuncs<phi::jit::SeqPoolTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
size_t n = ins.size();
size_t dst_step_size = n * w;
for (size_t i = 0; i < n; ++i) {
......
paddle/fluid/operators/fused/fusion_squared_mat_sub_op.cc
浏览文件 @ 2d36c9a9
......@@ -17,7 +17,7 @@
#include <string>
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -99,30 +99,30 @@ class FusionSquaredMatSubKernel : public framework::OpKernel<T> {
auto x_dims = x->dims();
auto y_dims = y->dims();
jit::matmul_attr_t attr;
phi::jit::matmul_attr_t attr;
attr.m = x_dims[0];
attr.k = x_dims[1];
attr.n = y_dims[1];
int o_numel = attr.m * attr.n;
auto vsquare_x =
jit::KernelFuncs<jit::VSquareTuple<T>, platform::CPUPlace>::Cache().At(
attr.m * attr.k);
auto vsquare_y =
jit::KernelFuncs<jit::VSquareTuple<T>, platform::CPUPlace>::Cache().At(
attr.k * attr.n);
auto vsquare_xy =
jit::KernelFuncs<jit::VSquareTuple<T>, platform::CPUPlace>::Cache().At(
o_numel);
auto vsub =
jit::KernelFuncs<jit::VSubTuple<T>, platform::CPUPlace>::Cache().At(
o_numel);
auto vscal =
jit::KernelFuncs<jit::VScalTuple<T>, platform::CPUPlace>::Cache().At(
o_numel);
auto matmul =
jit::KernelFuncs<jit::MatMulTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto vsquare_x = phi::jit::KernelFuncs<phi::jit::VSquareTuple<T>,
platform::CPUPlace>::Cache()
.At(attr.m * attr.k);
auto vsquare_y = phi::jit::KernelFuncs<phi::jit::VSquareTuple<T>,
platform::CPUPlace>::Cache()
.At(attr.k * attr.n);
auto vsquare_xy = phi::jit::KernelFuncs<phi::jit::VSquareTuple<T>,
platform::CPUPlace>::Cache()
.At(o_numel);
auto vsub = phi::jit::KernelFuncs<phi::jit::VSubTuple<T>,
platform::CPUPlace>::Cache()
.At(o_numel);
auto vscal = phi::jit::KernelFuncs<phi::jit::VScalTuple<T>,
platform::CPUPlace>::Cache()
.At(o_numel);
auto matmul = phi::jit::KernelFuncs<phi::jit::MatMulTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
const T* x_data = x->data<T>();
const T* y_data = y->data<T>();
......
paddle/fluid/operators/fused/multi_gru_op.cc
浏览文件 @ 2d36c9a9
......@@ -18,7 +18,6 @@ limitations under the License. */
#include <string>
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/fc_functor.h"
#include "paddle/phi/kernels/funcs/sequence2batch.h"
......
paddle/fluid/operators/math/sequence_pooling.cc
浏览文件 @ 2d36c9a9
......@@ -16,8 +16,8 @@ limitations under the License. */
#include <string>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace paddle {
......@@ -382,12 +382,12 @@ class SequencePoolFunctor<phi::CPUContext, T> {
"Sequence_pool should run on CPU Device when pooltype is SUM"));
const T* src = input.data<T>();
T* dst = output->mutable_data<T>(place);
jit::seq_pool_attr_t attr(
phi::jit::seq_pool_attr_t attr(
static_cast<int>(input.numel() / input.dims()[0]),
jit::SeqPoolType::kSum);
auto seqpool =
jit::KernelFuncs<jit::SeqPoolTuple<T>, platform::CPUPlace>::Cache()
.At(attr);
phi::jit::SeqPoolType::kSum);
auto seqpool = phi::jit::KernelFuncs<phi::jit::SeqPoolTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
for (int i = 0; i < static_cast<int>(lod.size()) - 1; ++i) {
attr.h = static_cast<int>(lod[i + 1] - lod[i]);
if (attr.h == 0) {
......
paddle/fluid/operators/optimizers/sgd_op.h
浏览文件 @ 2d36c9a9
......@@ -18,8 +18,8 @@ limitations under the License. */
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/framework/selected_rows_utils.h"
#include "paddle/fluid/framework/var_type_traits.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/fluid/platform/bfloat16.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace paddle {
namespace operators {
......@@ -43,16 +43,16 @@ struct sgd_dense_param_kernel<T,
const auto *grad = ctx.Input<phi::DenseTensor>("Grad");
const auto sz = param_out->numel();
jit::sgd_attr_t attr(1, sz, 1, sz, 1);
phi::jit::sgd_attr_t attr(1, sz, 1, sz, 1);
const T *lr = learning_rate->data<T>();
const T *param_data = param->data<T>();
const T *grad_data = grad->data<T>();
int64_t rows_idx = 0;
T *out_data = param_out->mutable_data<T>(ctx.GetPlace());
auto sgd =
jit::KernelFuncs<jit::SgdTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto sgd = phi::jit::KernelFuncs<phi::jit::SgdTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
sgd(lr, param_data, grad_data, &rows_idx, out_data, &attr);
}
};
......@@ -76,16 +76,16 @@ struct sgd_dense_param_kernel<T,
const int64_t *rows_data = grad_rows.data();
T *out_data = param_out->mutable_data<T>(ctx.GetPlace());
jit::sgd_attr_t attr;
phi::jit::sgd_attr_t attr;
attr.param_height = param_out->dims()[0];
attr.param_width = param_out->numel() / attr.param_height;
attr.grad_height = grad_rows.size(); // note: it is not grad->height()
attr.grad_width = grad_value.numel() / attr.grad_height;
attr.selected_rows_size = grad_rows.size();
auto sgd =
jit::KernelFuncs<jit::SgdTuple<T>, platform::CPUPlace>::Cache().At(
attr);
auto sgd = phi::jit::KernelFuncs<phi::jit::SgdTuple<T>,
platform::CPUPlace>::Cache()
.At(attr);
sgd(lr, param_data, grad_data, rows_data, out_data, &attr);
}
};
......
paddle/phi/kernels/cpu/adam_kernel.cc
浏览文件 @ 2d36c9a9
......@@ -16,11 +16,11 @@
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/tensor_utils.h"
#include "paddle/phi/kernels/funcs/adam_functors.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
DECLARE_int32(inner_op_parallelism);
......@@ -114,7 +114,7 @@ void AdamDenseKernel(const Context& dev_ctx,
learning_rate.data<T>()[0] * (sqrt(1 - beta2_p) / (1 - beta1_p));
T eps = epsilon_ * sqrt(1 - beta2_p);
paddle::operators::jit::adam_attr_t attr(beta1_, beta2_);
phi::jit::adam_attr_t attr(beta1_, beta2_);
int64_t numel = param.numel();
const T* param_ptr = param.data<T>();
......@@ -123,9 +123,8 @@ void AdamDenseKernel(const Context& dev_ctx,
const T* grad_ptr = grad.data<T>();
auto adam =
paddle::operators::jit::KernelFuncs<paddle::operators::jit::AdamTuple<T>,
phi::CPUPlace>::Cache()
.At(attr);
phi::jit::KernelFuncs<phi::jit::AdamTuple<T>, phi::CPUPlace>::Cache().At(
attr);
static constexpr int64_t chunk_size = 512;
......
paddle/phi/kernels/cpu/adamw_kernel.cc
浏览文件 @ 2d36c9a9
......@@ -16,13 +16,13 @@
#include <vector>
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/float16.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/tensor_utils.h"
#include "paddle/phi/kernels/adam_kernel.h"
#include "paddle/phi/kernels/funcs/adam_functors.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace phi {
......@@ -141,9 +141,8 @@ void AdamwDenseKernel(const Context& dev_ctx,
const T* grad_ptr = grad.data<T>();
auto adamw =
paddle::operators::jit::KernelFuncs<paddle::operators::jit::AdamWTuple<T>,
phi::CPUPlace>::Cache()
.At(1);
phi::jit::KernelFuncs<phi::jit::AdamWTuple<T>, phi::CPUPlace>::Cache().At(
1);
static constexpr int64_t chunk_size = 512;
......
paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc
浏览文件 @ 2d36c9a9
......@@ -18,7 +18,7 @@
#include "paddle/phi/kernels/funcs/layer_norm_util.h"
#if !defined(PADDLE_WITH_CUDA) && !defined(_WIN32) && !defined(__APPLE__) && \
!defined(__OSX__)
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#endif
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
......
paddle/phi/kernels/cpu/layer_norm_kernel.cc
浏览文件 @ 2d36c9a9
......@@ -18,7 +18,7 @@
#include "paddle/phi/kernels/funcs/layer_norm_util.h"
#if !defined(PADDLE_WITH_CUDA) && !defined(_WIN32) && !defined(__APPLE__) && \
!defined(__OSX__)
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#endif
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
......@@ -123,10 +123,9 @@ void LayerNormKernel(const Context& dev_ctx,
right));
}
auto ker = paddle::operators::jit::KernelFuncs<
paddle::operators::jit::LayerNormTuple<T>,
phi::CPUPlace>::Cache()
.At(right);
auto ker =
phi::jit::KernelFuncs<phi::jit::LayerNormTuple<T>, phi::CPUPlace>::Cache()
.At(right);
ker(x_tmp.data<T>(),
out.data<T>(),
mean->data<T>(),
......
paddle/phi/kernels/cpu/sgd_kernel.cc
浏览文件 @ 2d36c9a9
......@@ -14,10 +14,10 @@
#include "paddle/phi/kernels/sgd_kernel.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace phi {
......@@ -27,7 +27,7 @@ void sgd_dense_param_dense_grad_impl(const DenseTensor& param,
const DenseTensor& grad,
DenseTensor* param_out) {
const auto sz = param_out->numel();
paddle::operators::jit::sgd_attr_t attr(1, sz, 1, sz, 1);
phi::jit::sgd_attr_t attr(1, sz, 1, sz, 1);
const T* lr = learning_rate.data<T>();
const T* param_data = param.data<T>();
const T* grad_data = grad.data<T>();
......@@ -35,9 +35,8 @@ void sgd_dense_param_dense_grad_impl(const DenseTensor& param,
T* out_data = param_out->data<T>();
auto sgd =
paddle::operators::jit::KernelFuncs<paddle::operators::jit::SgdTuple<T>,
phi::CPUPlace>::Cache()
.At(attr);
phi::jit::KernelFuncs<phi::jit::SgdTuple<T>, phi::CPUPlace>::Cache().At(
attr);
sgd(lr, param_data, grad_data, &rows_idx, out_data, &attr);
}
......@@ -68,7 +67,7 @@ void sgd_dense_param_sparse_grad_impl(const DenseTensor& param,
const int64_t* rows_data = grad_rows.data();
T* out_data = param_out->data<T>();
paddle::operators::jit::sgd_attr_t attr;
phi::jit::sgd_attr_t attr;
attr.param_height = param_out->dims()[0];
attr.param_width = param_out->numel() / attr.param_height;
attr.grad_height = grad_rows.size(); // note: it is not grad->height()
......@@ -76,9 +75,8 @@ void sgd_dense_param_sparse_grad_impl(const DenseTensor& param,
attr.selected_rows_size = grad_rows.size();
auto sgd =
paddle::operators::jit::KernelFuncs<paddle::operators::jit::SgdTuple<T>,
phi::CPUPlace>::Cache()
.At(attr);
phi::jit::KernelFuncs<phi::jit::SgdTuple<T>, phi::CPUPlace>::Cache().At(
attr);
sgd(lr, param_data, grad_data, rows_data, out_data, &attr);
}
......
paddle/phi/kernels/funcs/CMakeLists.txt
浏览文件 @ 2d36c9a9
......@@ -2,6 +2,7 @@ add_subdirectory(eigen)
add_subdirectory(blas)
add_subdirectory(lapack)
add_subdirectory(detail)
add_subdirectory(jit)
math_library(deformable_conv_functor DEPS dense_tensor)
math_library(concat_and_split_functor DEPS dense_tensor)
......
paddle/phi/kernels/funcs/fc_functor.cc
浏览文件 @ 2d36c9a9
......@@ -14,9 +14,9 @@ limitations under the License. */
#include "paddle/phi/kernels/funcs/fc_functor.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/fluid/platform/device_context.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace phi {
namespace funcs {
......@@ -81,13 +81,11 @@ void FCFunctor<DeviceContext, T>::operator()(const DeviceContext& context,
errors::PermissionDenied("When bias is NULL, relu can not be true."));
return;
}
auto compute = relu ? paddle::operators::jit::KernelFuncs<
paddle::operators::jit::VAddReluTuple<T>,
paddle::platform::CPUPlace>::Cache()
auto compute = relu ? phi::jit::KernelFuncs<phi::jit::VAddReluTuple<T>,
phi::CPUPlace>::Cache()
.At(N)
: paddle::operators::jit::KernelFuncs<
paddle::operators::jit::VAddTuple<T>,
paddle::platform::CPUPlace>::Cache()
: phi::jit::KernelFuncs<phi::jit::VAddTuple<T>,
phi::CPUPlace>::Cache()
.At(N);
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for
......
paddle/fluid/operators/jit/CMakeLists.txt → paddle/phi/kernels/funcs/jit/CMakeLists.txt
浏览文件 @ 2d36c9a9
set(jit_file ${PADDLE_BINARY_DIR}/paddle/fluid/operators/jit/kernels.h.tmp)
set(jit_file_final ${PADDLE_BINARY_DIR}/paddle/fluid/operators/jit/kernels.h)
set(jit_file ${PADDLE_BINARY_DIR}/paddle/phi/kernels/funcs/jit/kernels.h.tmp)
set(jit_file_final ${PADDLE_BINARY_DIR}/paddle/phi/kernels/funcs/jit/kernels.h)
file(
WRITE ${jit_file}
"// Generated by the paddle/fluid/operators/jit/CMakeLists.txt. DO NOT EDIT!\n\n"
"// Generated by the paddle/phi/kernels/funcs/jit/CMakeLists.txt. DO NOT EDIT!\n\n"
)
file(APPEND ${jit_file} "\#pragma once\n")
file(APPEND ${jit_file} "\#include \"paddle/fluid/operators/jit/helper.h\"\n")
file(APPEND ${jit_file} "\#include \"paddle/phi/kernels/funcs/jit/helper.h\"\n")
file(APPEND ${jit_file}
"\#include \"paddle/fluid/operators/jit/registry.h\"\n\n")
"\#include \"paddle/phi/kernels/funcs/jit/registry.h\"\n\n")
set(JIT_KERNEL_DEPS device_context cblas gflags enforce place xxhash)
......
paddle/fluid/operators/jit/README.en.md → paddle/phi/kernels/funcs/jit/README.en.md
浏览文件 @ 2d36c9a9
......@@ -10,9 +10,9 @@ Currently it's only supported on CPU yet.
## Contents
```txt
PaddlePaddle/Paddle/paddle/fluid/
PaddlePaddle/Paddle/paddle/phi/kernels/
├── ...
└── operators/
└── funcs/
├── .../
└── jit/
├── ...
......@@ -34,7 +34,7 @@ PaddlePaddle/Paddle/paddle/fluid/
└── ...
```
All basical definitions of jit kernels are addressed in `paddle/fluid/operators/jit` including these three key folders `refer`, `gen`, `more`. There is only one unique name for each kernel while may have seraval implementations with same functionality.
All basical definitions of jit kernels are addressed in `paddle/phi/kernels/funcs/jit` including these three key folders `refer`, `gen`, `more`. There is only one unique name for each kernel while may have seraval implementations with same functionality.
- `refer`: Each kernel must have one reference implementation on CPU, and it should only focus on the correctness and should not depends on any third-party libraries.
- `gen`: The code generated should be kept here. They should be designed focusing on the best performance, which depends on Xbyak.
......@@ -55,7 +55,7 @@ Get from cache:
```cpp
using T = float;
jit::seq_pool_attr_t attr(width, jit::SeqPoolType::kSum);
auto seqpool_func = jit::KernelFuncs<jit::SeqPoolTuple<T>, platform::CPUPlace>::Cache().At(attr);
auto seqpool_func = jit::KernelFuncs<jit::SeqPoolTuple<T>, phi::CPUPlace>::Cache().At(attr);
seqpool_func(src_data, dst_data, &attr);
```
......@@ -64,14 +64,14 @@ Get all implementations and run once:
```cpp
using T = float;
jit::seq_pool_attr_t attr(width, jit::SeqPoolType::kSum);
auto funcs = jit::GetAllCandidateFuncsWithTypes<jit::SeqPoolTuple<T>, platform::CPUPlace>(attr);
auto funcs = jit::GetAllCandidateFuncsWithTypes<jit::SeqPoolTuple<T>, phi::CPUPlace>(attr);
for (auto f : funcs) {
LOG(INFO) << "Kernel implementation type: " << f.first;
f.second(src_data, dst_data, &attr);
}
```
All kernels are inlcuded in `paddle/fluid/operators/jit/kernels.h`, which is automatically generated in compile time, you can only include this one header to get all the registered kernels.
All kernels are inlcuded in `paddle/phi/kernels/funcs/jit/kernels.h`, which is automatically generated in compile time, you can only include this one header to get all the registered kernels.
## Solid Test
......
paddle/fluid/operators/jit/README.md → paddle/phi/kernels/funcs/jit/README.md
浏览文件 @ 2d36c9a9
......@@ -8,9 +8,9 @@
## 目录结构
```txt
PaddlePaddle/Paddle/paddle/fluid/
PaddlePaddle/Paddle/paddle/phi/kernels/
├── ...
└── operators/
└── funcs/
├── .../
└── jit/
├── ...
......@@ -46,14 +46,14 @@ PaddlePaddle/Paddle/paddle/fluid/
### 例子
所有kernel的调用只需要在头文件中包含`"paddle/fluid/operators/jit/kernels.h"`， 该文件是编译时自动生成的。
所有kernel的调用只需要在头文件中包含`"paddle/phi/kernels/funcs/jit/kernels.h"`， 该文件是编译时自动生成的。
直接从缓存中获取默认最优的函数。
```cpp
using T = float;
jit::seq_pool_attr_t attr(width, jit::SeqPoolType::kSum);
auto seqpool_func = jit::KernelFuncs<jit::SeqPoolTuple<T>, platform::CPUPlace>::Cache().At(attr);
auto seqpool_func = jit::KernelFuncs<jit::SeqPoolTuple<T>, phi::CPUPlace>::Cache().At(attr);
seqpool_func(src_data, dst_data, &attr);
```
......@@ -62,7 +62,7 @@ PaddlePaddle/Paddle/paddle/fluid/
```cpp
using T = float;
jit::seq_pool_attr_t attr(width, jit::SeqPoolType::kSum);
auto funcs = jit::GetAllCandidateFuncsWithTypes<jit::SeqPoolTuple<T>, platform::CPUPlace>(attr);
auto funcs = jit::GetAllCandidateFuncsWithTypes<jit::SeqPoolTuple<T>, phi::CPUPlace>(attr);
for (auto f : funcs) {
LOG(INFO) << "Kernel implementation type: " << f.first;
f.second(src_data, dst_data, &attr);
......
paddle/fluid/operators/jit/benchmark.cc → paddle/phi/kernels/funcs/jit/benchmark.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,11 @@
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/tensor.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/api/profiler/device_tracer.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
DEFINE_int32(burning, 10, "Burning times.");
DEFINE_int32(repeat, 3000, "Repeat times.");
......@@ -106,7 +106,7 @@ struct BenchFunc {
}
};
namespace jit = paddle::operators::jit;
namespace jit = phi::jit;
template <typename KernelTuple, typename PlaceType, typename... Args>
void BenchAllImpls(const typename KernelTuple::attr_type& attr, Args... args) {
......@@ -120,8 +120,7 @@ void BenchAllImpls(const typename KernelTuple::attr_type& attr, Args... args) {
// Test result from Get function
auto tgt = jit::KernelFuncs<KernelTuple, PlaceType>::Cache().At(attr);
if (!tgt) {
PADDLE_THROW(
paddle::platform::errors::Fatal("Benchmark target can not be empty."));
PADDLE_THROW(phi::errors::Fatal("Benchmark target can not be empty."));
}
infos.push_back(std::make_pair("Target", benchmark(tgt, args...)));
......@@ -323,7 +322,7 @@ void BenchKernelSgd() {
PADDLE_ENFORCE_LE(
static_cast<size_t>(upper - lower),
n - 1,
paddle::platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The range of Sgd (upper - lower) should be equal to or lower "
"than n-1 (Sgd size -1). But upper - lower is %d and n-1 is %d.",
static_cast<size_t>(upper - lower),
......@@ -331,7 +330,7 @@ void BenchKernelSgd() {
PADDLE_ENFORCE_GT(
n,
0,
paddle::platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The Sgd size should be larger than 0. But the n is %d.", n));
std::vector<int64_t> all, out;
for (int i = 0; i < n; ++i) {
......@@ -525,7 +524,7 @@ void BenchKernelVBroadcast() {
#define BenchKernelGRUHtPart1 BenchKernelGRU
#define BenchKernelGRUHtPart2 BenchKernelGRU
using CPUPlace = paddle::platform::CPUPlace;
using CPUPlace = phi::CPUPlace;
#define BENCH_FP32_CPU(name) \
BENCH_JITKERNEL(name, FP32, CPU) { \
......
paddle/fluid/operators/jit/gen/act.cc → paddle/phi/kernels/funcs/jit/gen/act.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,13 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/act.h"
#include "paddle/phi/kernels/funcs/jit/gen/act.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -150,10 +149,9 @@ size_t VTanhCreator::CodeSize(const int& d) const {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kVRelu, gen::VReluCreator);
REGISTER_JITKERNEL_GEN(kVSquare, gen::VSquareCreator);
......
paddle/fluid/operators/jit/gen/act.h → paddle/phi/kernels/funcs/jit/gen/act.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -91,7 +90,7 @@ class VActFunc : public JitCode {
int fy_idx = 13,
int mask_idx = 14,
int tmp_idx = 15) {
using namespace platform; // NOLINT
using namespace phi; // NOLINT
// check all idx can not equal
JMM jmm_src = JMM(src_idx);
JMM jmm_fx = JMM(fx_idx);
......@@ -266,7 +265,7 @@ class VActFunc : public JitCode {
identity_jmm<JMM>(dst, src, 15);
break;
default:
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support operand type code: %d.", type));
break;
}
......@@ -283,7 +282,7 @@ class VActJitCode : public VActFunc {
if (!(type_ == operand_type::RELU || type_ == operand_type::EXP ||
type_ == operand_type::SIGMOID || type_ == operand_type::TANH ||
type_ == operand_type::IDENTITY || type_ == operand_type::SQUARE)) {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support operand type code: %d.", type));
}
this->genCode();
......@@ -348,5 +347,4 @@ DECLARE_ACT_JITCODE(VTanh, operand_type::TANH);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/adam.cc → paddle/phi/kernels/funcs/jit/gen/adam.cc
浏览文件 @ 2d36c9a9
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/adam.h"
#include "paddle/phi/kernels/funcs/jit/gen/adam.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -145,9 +144,8 @@ class AdamCreator : public JitCodeCreator<adam_attr_t> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kAdam, gen::AdamCreator);
paddle/fluid/operators/jit/gen/adam.h → paddle/phi/kernels/funcs/jit/gen/adam.h
浏览文件 @ 2d36c9a9
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -72,5 +71,4 @@ class AdamJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/adamw.cc → paddle/phi/kernels/funcs/jit/gen/adamw.cc
浏览文件 @ 2d36c9a9
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/adamw.h"
#include "paddle/phi/kernels/funcs/jit/gen/adamw.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -157,9 +156,8 @@ class AdamWCreator : public JitCodeCreator<int> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kAdamW, gen::AdamWCreator);
paddle/fluid/operators/jit/gen/adamw.h → paddle/phi/kernels/funcs/jit/gen/adamw.h
浏览文件 @ 2d36c9a9
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -78,5 +77,4 @@ class AdamWJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/blas.cc → paddle/phi/kernels/funcs/jit/gen/blas.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/blas.h"
#include "paddle/phi/kernels/funcs/jit/gen/blas.h"
#include "paddle/fluid/operators/jit/macro.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/macro.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -179,10 +178,9 @@ DECLARE_BLAS_CREATOR(VAddBias);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kVMul, gen::VMulCreator);
REGISTER_JITKERNEL_GEN(kVAdd, gen::VAddCreator);
......
paddle/fluid/operators/jit/gen/blas.h → paddle/phi/kernels/funcs/jit/gen/blas.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -41,7 +40,7 @@ class VXXJitCode : public JitCode {
with_relu_(with_relu) {
if (!(type_ == operand_type::MUL || type_ == operand_type::ADD ||
type_ == operand_type::SUB)) {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support operand type code: %d.", type));
}
this->genCode();
......@@ -124,5 +123,4 @@ class NCHW16CMulNCJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/embseqpool.cc → paddle/phi/kernels/funcs/jit/gen/embseqpool.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,16 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/embseqpool.h"
#include "paddle/phi/kernels/funcs/jit/gen/embseqpool.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/macro.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/macro.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -133,31 +132,31 @@ class EmbSeqPoolCreator : public JitCodeCreator<emb_seq_pool_attr_t> {
const emb_seq_pool_attr_t& attr) const override {
PADDLE_ENFORCE_GT(attr.table_height,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute table_height of EmbSeqPool should "
"be larger than 0. But it is %d.",
attr.table_height));
PADDLE_ENFORCE_GT(attr.table_width,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute table_width of EmbSeqPool should "
"be larger than 0. But it is %d.",
attr.table_width));
PADDLE_ENFORCE_GT(attr.index_height,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute index_height of EmbSeqPool should "
"be larger than 0. But it is %d.",
attr.index_height));
PADDLE_ENFORCE_GT(attr.index_width,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute index_width of EmbSeqPool should "
"be larger than 0. But it is %d.",
attr.index_width));
PADDLE_ENFORCE_GT(attr.out_width,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute out_width of EmbSeqPool should be "
"larger than 0. But it is %d.",
attr.out_width));
......@@ -167,9 +166,8 @@ class EmbSeqPoolCreator : public JitCodeCreator<emb_seq_pool_attr_t> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kEmbSeqPool, gen::EmbSeqPoolCreator);
paddle/fluid/operators/jit/gen/embseqpool.h → paddle/phi/kernels/funcs/jit/gen/embseqpool.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -34,8 +33,7 @@ class EmbSeqPoolJitCode : public JitCode {
tbl_w_(attr.table_width),
type_(attr.pool_type) {
if (type_ != SeqPoolType::kSum) {
PADDLE_THROW(
platform::errors::Unimplemented("Only supports sum pool yet."));
PADDLE_THROW(phi::errors::Unimplemented("Only supports sum pool yet."));
}
this->genCode();
}
......@@ -79,5 +77,4 @@ class EmbSeqPoolJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/gru.cc → paddle/phi/kernels/funcs/jit/gen/gru.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,16 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/gru.h"
#include "paddle/phi/kernels/funcs/jit/gen/gru.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/macro.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/macro.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -110,10 +109,9 @@ DECLARE_GRU_CREATOR(GRUHtPart2);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kGRUH1, gen::GRUH1Creator);
REGISTER_JITKERNEL_GEN(kGRUHtPart1, gen::GRUHtPart1Creator);
......
paddle/fluid/operators/jit/gen/gru.h → paddle/phi/kernels/funcs/jit/gen/gru.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/act.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/phi/kernels/funcs/jit/gen/act.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -42,7 +41,7 @@ class GRUJitCode : public VActFunc {
} else if (type == KernelType::kVIdentity) {
return operand_type::IDENTITY;
} else {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support jit::KernelType code: %d.", type));
}
return operand_type::IDENTITY;
......@@ -114,5 +113,4 @@ DECLARE_GRU_JITCODE(GRUHtPart2, 2);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/hopv.cc → paddle/phi/kernels/funcs/jit/gen/hopv.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,13 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/hopv.h"
#include "paddle/phi/kernels/funcs/jit/gen/hopv.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -95,10 +94,9 @@ DECLARE_HOP_CREATOR(HSum);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kHMax, gen::HMaxCreator);
REGISTER_JITKERNEL_GEN(kHSum, gen::HSumCreator);
paddle/fluid/operators/jit/gen/hopv.h → paddle/phi/kernels/funcs/jit/gen/hopv.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -34,7 +33,7 @@ class HOPVJitCode : public JitCode {
void* code_ptr = nullptr)
: JitCode(code_size, code_ptr), num_(d), type_(type) {
if (!(type_ == operand_type::MAX || type_ == operand_type::ADD)) {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support operand type code: %d.", type));
}
this->genCode();
......@@ -91,5 +90,4 @@ DECLARE_HOP_JITCODE(HSum, operand_type::ADD);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/jitcode.h → paddle/phi/kernels/funcs/jit/gen/jitcode.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,15 +17,14 @@
#include <string>
#include <type_traits>
#include "paddle/fluid/operators/jit/gen_base.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/gen_base.h"
#define XBYAK_USE_MMAP_ALLOCATOR
#include "xbyak/xbyak.h"
#include "xbyak/xbyak_util.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -131,5 +130,4 @@ class JitCode : public GenBase, public Xbyak::CodeGenerator {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/lstm.cc → paddle/phi/kernels/funcs/jit/gen/lstm.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,16 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/lstm.h"
#include "paddle/phi/kernels/funcs/jit/gen/lstm.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/macro.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/macro.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -137,10 +136,9 @@ DECLARE_LSTM_CREATOR(LSTMC1H1);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kLSTMCtHt, gen::LSTMCtHtCreator);
REGISTER_JITKERNEL_GEN(kLSTMC1H1, gen::LSTMC1H1Creator);
paddle/fluid/operators/jit/gen/lstm.h → paddle/phi/kernels/funcs/jit/gen/lstm.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/act.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/phi/kernels/funcs/jit/gen/act.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -45,7 +44,7 @@ class LSTMJitCode : public VActFunc {
} else if (type == KernelType::kVIdentity) {
return operand_type::IDENTITY;
} else {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Do not support jit::KernelType code: %d.", type));
}
return operand_type::IDENTITY;
......@@ -119,5 +118,4 @@ DECLARE_LSTM_JITCODE(LSTMC1H1, true);
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/matmul.cc → paddle/phi/kernels/funcs/jit/gen/matmul.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/matmul.h"
#include "paddle/phi/kernels/funcs/jit/gen/matmul.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -31,9 +30,9 @@ void MatMulJitCode::genCode() {
PADDLE_ENFORCE_GT(
groups.front(),
0,
platform::errors::InvalidArgument("The number of rest registers should "
"be larger than 0. But it is %d.",
groups.front()));
phi::errors::InvalidArgument("The number of rest registers should "
"be larger than 0. But it is %d.",
groups.front()));
const int block_len = sizeof(float) * block;
const int x_reg_idx = (block == ZMM_FLOAT_BLOCK ? 32 : 16) - 1;
......@@ -126,21 +125,21 @@ class MatMulCreator : public JitCodeCreator<matmul_attr_t> {
PADDLE_ENFORCE_GT(
attr.m,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute m (first matrix's row) of MatMul should "
"be larger than 0. But it is %d.",
attr.m));
PADDLE_ENFORCE_GT(
attr.n,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute n (first matrix's col) of MatMul should "
"be larger than 0. But it is %d.",
attr.n));
PADDLE_ENFORCE_GT(
attr.k,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute k (second matrix's col) of MatMul should "
"be larger than 0. But it is %d.",
attr.k));
......@@ -150,9 +149,8 @@ class MatMulCreator : public JitCodeCreator<matmul_attr_t> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kMatMul, gen::MatMulCreator);
paddle/fluid/operators/jit/gen/matmul.h → paddle/phi/kernels/funcs/jit/gen/matmul.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -20,11 +20,10 @@
#include <vector>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -34,12 +33,12 @@ class MatMulJitCode : public JitCode {
size_t code_size = 256 * 1024,
void* code_ptr = nullptr)
: JitCode(code_size, code_ptr), m_(attr.m), n_(attr.n), k_(attr.k) {
PADDLE_ENFORCE_EQ(m_,
1,
platform::errors::Unimplemented(
"Jitcode of matmul only support m==1 (first "
"matrix's row) now. But m is %d.",
m_));
PADDLE_ENFORCE_EQ(
m_,
1,
phi::errors::Unimplemented("Jitcode of matmul only support m==1 (first "
"matrix's row) now. But m is %d.",
m_));
this->genCode();
}
......@@ -65,5 +64,4 @@ class MatMulJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/seqpool.cc → paddle/phi/kernels/funcs/jit/gen/seqpool.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/seqpool.h"
#include "paddle/phi/kernels/funcs/jit/gen/seqpool.h"
#include "paddle/fluid/operators/jit/gen/act.h" // for exp_float_consts ones
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/gen/act.h" // for exp_float_consts ones
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -69,27 +68,26 @@ class SeqPoolCreator : public JitCodeCreator<seq_pool_attr_t> {
}
std::unique_ptr<GenBase> CreateJitCode(
const seq_pool_attr_t& attr) const override {
PADDLE_ENFORCE_GT(attr.w,
0,
platform::errors::InvalidArgument(
"The attribute width of SeqPool should "
"be larger than 0. But it is %d.",
attr.w));
PADDLE_ENFORCE_GT(attr.h,
0,
platform::errors::InvalidArgument(
"The attribute height of SeqPool should "
"be larger than 0. But it is %d.",
attr.h));
PADDLE_ENFORCE_GT(
attr.w,
0,
phi::errors::InvalidArgument("The attribute width of SeqPool should "
"be larger than 0. But it is %d.",
attr.w));
PADDLE_ENFORCE_GT(
attr.h,
0,
phi::errors::InvalidArgument("The attribute height of SeqPool should "
"be larger than 0. But it is %d.",
attr.h));
return make_unique<SeqPoolJitCode>(attr, CodeSize(attr));
}
};
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kSeqPool, gen::SeqPoolCreator);
paddle/fluid/operators/jit/gen/seqpool.h → paddle/phi/kernels/funcs/jit/gen/seqpool.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -33,7 +32,7 @@ class SeqPoolJitCode : public JitCode {
: JitCode(code_size, code_ptr), w_(attr.w), type_(attr.type) {
if (!(type_ == SeqPoolType::kSum || type_ == SeqPoolType::kAvg ||
type_ == SeqPoolType::kSqrt)) {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Only supports sum, average and sqrt pool type."));
}
fp_h_[0] = 1.f;
......@@ -130,7 +129,7 @@ class SeqPoolJitCode : public JitCode {
PADDLE_ENFORCE_EQ(
reg_idx,
rest_used_num_regs,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"All heights of SeqPool should use the same number of registers."
"It equals to the numbr of rest registers. But use %d registers "
"and the numbr of rest registers is %d.",
......@@ -221,5 +220,4 @@ class SeqPoolJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/sgd.cc → paddle/phi/kernels/funcs/jit/gen/sgd.cc
浏览文件 @ 2d36c9a9
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/sgd.h"
#include "paddle/phi/kernels/funcs/jit/gen/sgd.h"
#include <stddef.h> // offsetof
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -117,7 +116,7 @@ class SgdCreator : public JitCodeCreator<sgd_attr_t> {
const sgd_attr_t& attr) const override {
PADDLE_ENFORCE_EQ(attr.param_width,
attr.grad_width,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute param_width of Sgd should be "
"equal to the attribute grad_width. But param_width "
"is %d and grad_width is %d.",
......@@ -125,7 +124,7 @@ class SgdCreator : public JitCodeCreator<sgd_attr_t> {
attr.grad_width));
PADDLE_ENFORCE_LE(attr.selected_rows_size,
attr.grad_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute selected_rows_size of Sgd should be "
"equal to or less than the attribute grad_height. "
"But selected_rows_size is %d and grad_height is %d.",
......@@ -134,7 +133,7 @@ class SgdCreator : public JitCodeCreator<sgd_attr_t> {
PADDLE_ENFORCE_GE(
attr.selected_rows_size,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute selected_rows_size of Sgd should be "
"equal to or larger than 0. But selected_rows_size is %d.",
attr.selected_rows_size));
......@@ -144,9 +143,8 @@ class SgdCreator : public JitCodeCreator<sgd_attr_t> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kSgd, gen::SgdCreator);
paddle/fluid/operators/jit/gen/sgd.h → paddle/phi/kernels/funcs/jit/gen/sgd.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,11 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -59,5 +58,4 @@ class SgdJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen/vbroadcast.cc → paddle/phi/kernels/funcs/jit/gen/vbroadcast.cc
浏览文件 @ 2d36c9a9
......@@ -12,13 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen/vbroadcast.h"
#include "paddle/phi/kernels/funcs/jit/gen/vbroadcast.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -79,7 +78,7 @@ class VBroadcastCreator : public JitCodeCreator<int64_t> {
PADDLE_ENFORCE_GT(
w,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The width of VBroadcast should be larger than 0. But w is %d.",
w));
return make_unique<VBroadcastJitCode>(w, CodeSize(w));
......@@ -88,9 +87,8 @@ class VBroadcastCreator : public JitCodeCreator<int64_t> {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace gen = paddle::operators::jit::gen;
namespace gen = phi::jit::gen;
REGISTER_JITKERNEL_GEN(kVBroadcast, gen::VBroadcastCreator);
paddle/fluid/operators/jit/gen/vbroadcast.h → paddle/phi/kernels/funcs/jit/gen/vbroadcast.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,10 +17,10 @@
#include <string>
#include "glog/logging.h"
#include "paddle/fluid/operators/jit/gen/jitcode.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen/jitcode.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace gen {
......@@ -51,5 +51,4 @@ class VBroadcastJitCode : public JitCode {
} // namespace gen
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen_base.cc → paddle/phi/kernels/funcs/jit/gen_base.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,22 +12,22 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/gen_base.h"
#include "paddle/phi/kernels/funcs/jit/gen_base.h"
#include <fstream>
#include "paddle/fluid/memory/allocation/cpu_allocator.h" // for posix_memalign
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/core/enforce.h"
#ifndef _WIN32
#ifdef _WIN32
#define posix_memalign_free _aligned_free
#else
#define posix_memalign_free free
#endif
DEFINE_bool(dump_jitcode, false, "Whether to dump the jitcode to file");
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
// refer do not need CanBeUsed, it would be the last one.
......@@ -48,16 +48,20 @@ void GenBase::dumpCode(const unsigned char* code) const {
void* GenBase::operator new(size_t size) {
void* ptr;
constexpr size_t alignment = 32ul;
#ifdef _WIN32
ptr = _aligned_malloc(size, alignment);
#else
PADDLE_ENFORCE_EQ(
posix_memalign(&ptr, alignment, size),
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"Jitcode generator (GenBase) allocate %ld memory error!", size));
#endif
PADDLE_ENFORCE_NOT_NULL(
ptr,
platform::errors::InvalidArgument("Fail to allocate jitcode generator "
"(GenBase) CPU memory: size = %d .",
size));
phi::errors::InvalidArgument("Fail to allocate jitcode generator "
"(GenBase) CPU memory: size = %d .",
size));
return ptr;
}
......@@ -93,5 +97,4 @@ std::vector<int> packed_groups(int n, int k, int* block_out, int* rest_out) {
}
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/gen_base.h → paddle/phi/kernels/funcs/jit/gen_base.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -18,13 +18,16 @@
#include <string>
#include <vector>
#ifdef _WIN32
#include <malloc.h> // for _aligned_malloc
#endif
#include "gflags/gflags.h"
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
DECLARE_bool(dump_jitcode);
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
class GenBase : public Kernel {
......@@ -84,5 +87,4 @@ std::vector<int> packed_groups(int n,
int* rest = nullptr);
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/helper.cc → paddle/phi/kernels/funcs/jit/helper.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/helper.h"
#include "paddle/phi/kernels/funcs/jit/helper.h"
#include <numeric>
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
std::map<size_t, std::shared_ptr<void>>& GetFuncCacheMap() {
......@@ -68,7 +67,7 @@ const char* to_string(KernelType kt) {
ONE_CASE(kEmbSeqPool);
ONE_CASE(kSgd);
default:
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"JIT kernel do not support type: %d.", kt));
return "NOT JITKernel";
}
......@@ -82,7 +81,7 @@ const char* to_string(SeqPoolType tp) {
ONE_CASE(kAvg);
ONE_CASE(kSqrt);
default:
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"SeqPool JIT kernel do not support type: %d.", tp));
return "NOT PoolType";
}
......@@ -104,7 +103,7 @@ KernelType to_kerneltype(const std::string& act) {
} else if (lower == "tanh" || lower == "vtanh") {
return kVTanh;
}
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Act JIT kernel do not support type: %s.", act));
return kNone;
}
......@@ -116,7 +115,7 @@ void pack_weights<float>(const float* src, float* dst, int n, int k) {
std::for_each(groups.begin(), groups.end(), [&](int i) {
PADDLE_ENFORCE_GT(i,
0,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"Each element of groups should be larger than "
"0. However the element: %d doesn't satify.",
i));
......@@ -125,7 +124,7 @@ void pack_weights<float>(const float* src, float* dst, int n, int k) {
std::memset(dst, 0, k * sum * block * sizeof(float));
PADDLE_ENFORCE_GE(sum * block,
n,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The packed n (sum * block) should be equal to or "
"larger than n (matmul row size). "
"However, the packed n is %d and n is %d.",
......@@ -152,10 +151,9 @@ void pack_weights<float>(const float* src, float* dst, int n, int k) {
template <typename T>
typename std::enable_if<!std::is_same<T, float>::value>::type pack_weights(
const T* src, T* dst, int n, int k) {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Only supports pack weights with float type."));
}
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/helper.h → paddle/phi/kernels/funcs/jit/helper.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -14,22 +14,22 @@
#pragma once
#include <cstring>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility> // for std::move
#include <vector>
#include "paddle/fluid/operators/jit/gen_base.h"
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/operators/jit/kernel_key.h"
#include "paddle/fluid/operators/jit/kernel_pool.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/gen_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_key.h"
#include "paddle/phi/kernels/funcs/jit/kernel_pool.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
class GenBase;
......@@ -37,7 +37,7 @@ class GenBase;
template <typename KernelTuple, typename PlaceType>
inline typename std::enable_if<
std::is_same<typename KernelTuple::data_type, float>::value &&
std::is_same<PlaceType, platform::CPUPlace>::value,
std::is_same<PlaceType, phi::CPUPlace>::value,
const Kernel*>::type
GetJitCode(const typename KernelTuple::attr_type& attr) {
using Attr = typename KernelTuple::attr_type;
......@@ -72,7 +72,7 @@ GetJitCode(const typename KernelTuple::attr_type& attr) {
template <typename KernelTuple, typename PlaceType>
inline typename std::enable_if<
!std::is_same<typename KernelTuple::data_type, float>::value ||
!std::is_same<PlaceType, platform::CPUPlace>::value,
!std::is_same<PlaceType, phi::CPUPlace>::value,
const Kernel*>::type
GetJitCode(const typename KernelTuple::attr_type& attr) {
return nullptr;
......@@ -83,12 +83,12 @@ GetJitCode(const typename KernelTuple::attr_type& attr) {
template <typename KernelTuple>
inline const Kernel* GetReferKernel() {
auto& ref_pool = ReferKernelPool::Instance().AllKernels();
KernelKey kkey(KernelTuple::kernel_type, platform::CPUPlace());
KernelKey kkey(KernelTuple::kernel_type, phi::CPUPlace());
auto ref_iter = ref_pool.find(kkey);
PADDLE_ENFORCE_NE(
ref_iter,
ref_pool.end(),
platform::errors::PreconditionNotMet(
phi::errors::PreconditionNotMet(
"Every Refer Kernel of jitcode should have reference function."));
auto& ref_impls = ref_iter->second;
for (auto& impl : ref_impls) {
......@@ -104,10 +104,10 @@ template <typename KernelTuple>
inline typename KernelTuple::func_type GetReferFunc() {
auto ker = GetReferKernel<KernelTuple>();
auto p = dynamic_cast<const ReferKernel<KernelTuple>*>(ker);
PADDLE_ENFORCE_NOT_NULL(p,
platform::errors::InvalidArgument(
"Get the reference code of kernel in CPU "
"failed. The Refer kernel should exsit."));
PADDLE_ENFORCE_NOT_NULL(
p,
phi::errors::InvalidArgument("Get the reference code of kernel in CPU "
"failed. The Refer kernel should exsit."));
return p->GetFunc();
}
......@@ -138,15 +138,15 @@ std::vector<const Kernel*> GetAllCandidateKernels(
// The last implementation should be reference function on CPUPlace.
auto ref = GetReferKernel<KernelTuple>();
PADDLE_ENFORCE_NOT_NULL(ref,
platform::errors::InvalidArgument(
"Get all candicate kernel in CPU failed. "
"The Refer Kernel can not be empty."));
PADDLE_ENFORCE_NOT_NULL(
ref,
phi::errors::InvalidArgument("Get all candicate kernel in CPU failed. "
"The Refer Kernel can not be empty."));
res.emplace_back(ref);
return res;
}
template <typename KernelTuple, typename PlaceType = platform::CPUPlace>
template <typename KernelTuple, typename PlaceType = phi::CPUPlace>
std::vector<std::pair<std::string, typename KernelTuple::func_type>>
GetAllCandidateFuncsWithTypes(const typename KernelTuple::attr_type& attr) {
using Func = typename KernelTuple::func_type;
......@@ -157,21 +157,20 @@ GetAllCandidateFuncsWithTypes(const typename KernelTuple::attr_type& attr) {
if (name == "JitCode") {
auto i = dynamic_cast<const GenBase*>(k);
PADDLE_ENFORCE_NOT_NULL(i,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"Generate jitcode kernel (GenBase) failed."));
res.emplace_back(std::make_pair(name, i->template getCode<Func>()));
} else {
auto i = dynamic_cast<const KernelMore<KernelTuple>*>(k);
PADDLE_ENFORCE_NOT_NULL(i,
platform::errors::InvalidArgument(
"Kernel cast (KernelMore) failed."));
PADDLE_ENFORCE_NOT_NULL(
i, phi::errors::InvalidArgument("Kernel cast (KernelMore) failed."));
res.emplace_back(std::make_pair(name, i->GetFunc()));
}
}
return res;
}
template <typename KernelTuple, typename PlaceType = platform::CPUPlace>
template <typename KernelTuple, typename PlaceType = phi::CPUPlace>
std::vector<typename KernelTuple::func_type> GetAllCandidateFuncs(
const typename KernelTuple::attr_type& attr) {
auto funcs = GetAllCandidateFuncsWithTypes<KernelTuple, PlaceType>(attr);
......@@ -182,13 +181,13 @@ std::vector<typename KernelTuple::func_type> GetAllCandidateFuncs(
return res;
}
template <typename KernelTuple, typename PlaceType = platform::CPUPlace>
template <typename KernelTuple, typename PlaceType = phi::CPUPlace>
typename KernelTuple::func_type GetDefaultBestFunc(
const typename KernelTuple::attr_type& attr) {
auto funcs = GetAllCandidateFuncs<KernelTuple, PlaceType>(attr);
PADDLE_ENFORCE_GE(funcs.size(),
1UL,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The candicate jit kernel is at least one in CPU."));
// Here could do some runtime benchmark of this attr and return the best one.
// But yet just get the first one as the default best one,
......@@ -303,5 +302,4 @@ template <typename T>
void pack_weights(const T* src, T* dst, int n, int k);
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/kernel_base.h → paddle/phi/kernels/funcs/jit/kernel_base.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -15,11 +15,10 @@
#pragma once
#include <cstdint>
#include "paddle/fluid/operators/jit/macro.h"
#include "paddle/fluid/platform/macros.h"
#include "paddle/phi/core/macros.h"
#include "paddle/phi/kernels/funcs/jit/macro.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
typedef enum {
......@@ -403,5 +402,4 @@ class ReferKernel : public KernelMore<KernelTuple> {
};
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/kernel_key.cc → paddle/phi/kernels/funcs/jit/kernel_key.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,12 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/kernel_key.h"
#include "paddle/phi/kernels/funcs/jit/kernel_key.h"
#include <xxhash.h> // XXH64: 13.8 GB/s
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
template <>
......@@ -72,5 +71,4 @@ int64_t JitCodeKey<adam_attr_t>(const adam_attr_t& attr) {
}
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/kernel_key.h → paddle/phi/kernels/funcs/jit/kernel_key.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -13,11 +13,10 @@
* limitations under the License. */
#pragma once
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
struct KernelKey {
......@@ -31,15 +30,13 @@ struct KernelKey {
};
KernelType type_;
platform::Place place_;
phi::Place place_;
KernelKey(KernelType type, platform::Place place)
: type_(type), place_(place) {}
KernelKey(KernelType type, phi::Place place) : type_(type), place_(place) {}
size_t hash_key() const { return Hash()(*this); }
bool operator==(const KernelKey& o) const {
return platform::places_are_same_class(place_, o.place_) &&
type_ == o.type_;
return place_ == o.place_ && type_ == o.type_;
}
bool operator!=(const KernelKey& o) const { return !(*this == o); }
};
......@@ -49,5 +46,4 @@ template <typename Attr>
int64_t JitCodeKey(const Attr& attr);
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/kernel_pool.cc → paddle/phi/kernels/funcs/jit/kernel_pool.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,10 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/kernel_pool.h"
#include "paddle/phi/kernels/funcs/jit/kernel_pool.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
std::map<size_t, std::shared_ptr<void>>& GetJITCodesMap() {
......@@ -39,5 +38,4 @@ ReferKernelPool& ReferKernelPool::Instance() {
}
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/kernel_pool.h → paddle/phi/kernels/funcs/jit/kernel_pool.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -21,13 +21,12 @@
#include <utility> // for move
#include <vector>
#include "paddle/fluid/operators/jit/gen_base.h"
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/operators/jit/kernel_key.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/kernels/funcs/jit/gen_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_key.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
struct KernelKey;
......@@ -130,5 +129,4 @@ class ReferKernelPool {
};
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/macro.h → paddle/phi/kernels/funcs/jit/macro.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -15,8 +15,7 @@
#pragma once
#include <type_traits>
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
#define SIGMOID_THRESHOLD_MIN -40.0
......@@ -28,5 +27,4 @@ namespace jit {
#define ZMM_FLOAT_BLOCK 16
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/more/intrinsic/crf_decoding.cc → paddle/phi/kernels/funcs/jit/more/intrinsic/crf_decoding.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/more/intrinsic/crf_decoding.h"
#include "paddle/phi/kernels/funcs/jit/more/intrinsic/crf_decoding.h"
#include <limits>
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace intrinsic {
......@@ -178,9 +177,8 @@ bool CRFDecodingKernel::CanBeUsed(const int& d) const {
} // namespace intrinsic
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace intrinsic = paddle::operators::jit::more::intrinsic;
namespace intrinsic = phi::jit::more::intrinsic;
REGISTER_JITKERNEL_MORE(kCRFDecoding, intrinsic, intrinsic::CRFDecodingKernel);
paddle/fluid/operators/jit/more/intrinsic/crf_decoding.h → paddle/phi/kernels/funcs/jit/more/intrinsic/crf_decoding.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -16,10 +16,9 @@
#include <type_traits>
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace intrinsic {
......@@ -42,5 +41,4 @@ class CRFDecodingKernel : public KernelMore<CRFDecodingTuple<float>> {
} // namespace intrinsic
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/more/intrinsic/layer_norm.cc → paddle/phi/kernels/funcs/jit/more/intrinsic/layer_norm.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/more/intrinsic/layer_norm.h"
#include "paddle/phi/kernels/funcs/jit/more/intrinsic/layer_norm.h"
#include <limits>
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace intrinsic {
......@@ -186,9 +185,8 @@ bool LayerNormKernel::CanBeUsed(const int& d) const {
} // namespace intrinsic
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace intrinsic = paddle::operators::jit::more::intrinsic;
namespace intrinsic = phi::jit::more::intrinsic;
REGISTER_JITKERNEL_MORE(kLayerNorm, intrinsic, intrinsic::LayerNormKernel);
paddle/fluid/operators/jit/more/intrinsic/layer_norm.h → paddle/phi/kernels/funcs/jit/more/intrinsic/layer_norm.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -16,10 +16,9 @@
#include <type_traits>
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace intrinsic {
......@@ -45,5 +44,4 @@ class LayerNormKernel : public KernelMore<LayerNormTuple<float>> {
} // namespace intrinsic
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/more/mix/mix.cc → paddle/phi/kernels/funcs/jit/more/mix/mix.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,18 +12,17 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/more/mix/mix.h"
#include "paddle/phi/kernels/funcs/jit/more/mix/mix.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace mix {
using CPUPlace = platform::CPUPlace;
using CPUPlace = phi::CPUPlace;
void VSigmoid(const T* x, T* y, int n) {
const float min = SIGMOID_THRESHOLD_MIN;
......@@ -95,7 +94,7 @@ void (*getActFunc(KernelType type, int d))(const T*, T*, int) { // NOLINT
} else if (type == kVIdentity) {
return KernelFuncs<VIdentityTuple<T>, CPUPlace>::Cache().At(d);
}
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Act JIT kernel do not support type: %s", type));
return nullptr;
}
......@@ -237,10 +236,9 @@ bool GRUHtPart2Kernel::CanBeUsed(const gru_attr_t& attr) const { return true; }
} // namespace mix
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace mix = paddle::operators::jit::more::mix;
namespace mix = phi::jit::more::mix;
#define REGISTER_MORE_KERNEL(func) \
REGISTER_JITKERNEL_MORE(k##func, mix, mix::func##Kernel)
......
paddle/fluid/operators/jit/more/mix/mix.h → paddle/phi/kernels/funcs/jit/more/mix/mix.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -16,10 +16,9 @@
#include <type_traits>
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace mix {
......@@ -62,5 +61,4 @@ DECLARE_MORE_KERNEL(GRUHtPart2);
} // namespace mix
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/more/mkl/mkl.cc → paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,15 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/more/mkl/mkl.h"
#include "paddle/phi/kernels/funcs/jit/more/mkl/mkl.h"
#include "paddle/fluid/operators/jit/refer/refer.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/fluid/platform/dynload/mklml.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/backends/dynload/mklml.h"
#include "paddle/phi/kernels/funcs/jit/refer/refer.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace mkl {
......@@ -30,20 +29,20 @@ void MatMul<float>(const float* a,
const float* b,
float* c,
const matmul_attr_t* attr) {
platform::dynload::cblas_sgemm(CblasRowMajor,
CblasNoTrans,
CblasNoTrans,
attr->m,
attr->n,
attr->k,
1.f,
a,
attr->k,
b,
attr->n,
0.f,
c,
attr->n);
phi::dynload::cblas_sgemm(CblasRowMajor,
CblasNoTrans,
CblasNoTrans,
attr->m,
attr->n,
attr->k,
1.f,
a,
attr->k,
b,
attr->n,
0.f,
c,
attr->n);
}
template <>
......@@ -51,46 +50,46 @@ void MatMul<double>(const double* a,
const double* b,
double* c,
const matmul_attr_t* attr) {
platform::dynload::cblas_dgemm(CblasRowMajor,
CblasNoTrans,
CblasNoTrans,
attr->m,
attr->n,
attr->k,
1.0,
a,
attr->k,
b,
attr->n,
0.0,
c,
attr->n);
phi::dynload::cblas_dgemm(CblasRowMajor,
CblasNoTrans,
CblasNoTrans,
attr->m,
attr->n,
attr->k,
1.0,
a,
attr->k,
b,
attr->n,
0.0,
c,
attr->n);
}
template <>
void VMul<float>(const float* x, const float* y, float* z, int n) {
platform::dynload::vsMul(n, x, y, z);
phi::dynload::vsMul(n, x, y, z);
}
template <>
void VMul<double>(const double* x, const double* y, double* z, int n) {
platform::dynload::vdMul(n, x, y, z);
phi::dynload::vdMul(n, x, y, z);
}
template <>
void VAdd<float>(const float* x, const float* y, float* z, int n) {
platform::dynload::vsAdd(n, x, y, z);
phi::dynload::vsAdd(n, x, y, z);
}
template <>
void VAdd<double>(const double* x, const double* y, double* z, int n) {
platform::dynload::vdAdd(n, x, y, z);
phi::dynload::vdAdd(n, x, y, z);
}
template <>
void VScal<float>(const float* a, const float* x, float* y, int n) {
if (x == y) {
platform::dynload::cblas_sscal(n, *a, y, 1);
phi::dynload::cblas_sscal(n, *a, y, 1);
} else {
refer::VScal<float>(a, x, y, n);
}
......@@ -99,7 +98,7 @@ void VScal<float>(const float* a, const float* x, float* y, int n) {
template <>
void VScal<double>(const double* a, const double* x, double* y, int n) {
if (x == y) {
platform::dynload::cblas_dscal(n, *a, y, 1);
phi::dynload::cblas_dscal(n, *a, y, 1);
} else {
refer::VScal<double>(a, x, y, n);
}
......@@ -109,7 +108,7 @@ template <>
void StrideScal<float>(
const float* a, const float* x, float* y, int n, int stride) {
if (x == y) {
platform::dynload::cblas_sscal(n / stride, *a, y, stride);
phi::dynload::cblas_sscal(n / stride, *a, y, stride);
} else {
refer::StrideScal<float>(a, x, y, n, stride);
}
......@@ -119,7 +118,7 @@ template <>
void StrideScal<double>(
const double* a, const double* x, double* y, int n, int stride) {
if (x == y) {
platform::dynload::cblas_dscal(n / stride, *a, y, stride);
phi::dynload::cblas_dscal(n / stride, *a, y, stride);
} else {
refer::StrideScal<double>(a, x, y, n, stride);
}
......@@ -127,62 +126,62 @@ void StrideScal<double>(
template <>
void VExp<float>(const float* x, float* y, int n) {
platform::dynload::vsExp(n, x, y);
phi::dynload::vsExp(n, x, y);
}
template <>
void VExp<double>(const double* x, double* y, int n) {
platform::dynload::vdExp(n, x, y);
phi::dynload::vdExp(n, x, y);
}
template <>
void VSquare<float>(const float* x, float* y, int n) {
platform::dynload::vsSqr(n, x, y);
phi::dynload::vsSqr(n, x, y);
}
template <>
void VSquare<double>(const double* x, double* y, int n) {
platform::dynload::vdSqr(n, x, y);
phi::dynload::vdSqr(n, x, y);
}
template <>
void VCopy<float>(const float* x, float* y, int n) {
platform::dynload::cblas_scopy(n, x, 1, y, 1);
phi::dynload::cblas_scopy(n, x, 1, y, 1);
}
template <>
void VCopy<double>(const double* x, double* y, int n) {
platform::dynload::cblas_dcopy(n, x, 1, y, 1);
phi::dynload::cblas_dcopy(n, x, 1, y, 1);
}
template <>
void VAXPY<float>(float a, const float* x, float* y, int n) {
platform::dynload::cblas_saxpy(n, a, x, 1, y, 1);
phi::dynload::cblas_saxpy(n, a, x, 1, y, 1);
}
template <>
void VAXPY<double>(double a, const double* x, double* y, int n) {
platform::dynload::cblas_daxpy(n, a, x, 1, y, 1);
phi::dynload::cblas_daxpy(n, a, x, 1, y, 1);
}
template <>
void ASum<float>(const float* x, float* res, int n) {
res[0] = platform::dynload::cblas_sasum(n, x, 1);
res[0] = phi::dynload::cblas_sasum(n, x, 1);
}
template <>
void ASum<double>(const double* x, double* res, int n) {
res[0] = platform::dynload::cblas_dasum(n, x, 1);
res[0] = phi::dynload::cblas_dasum(n, x, 1);
}
template <>
void StrideASum<float>(const float* x, float* res, int n, int stride) {
res[0] = platform::dynload::cblas_sasum(n / stride, x, stride);
res[0] = phi::dynload::cblas_sasum(n / stride, x, stride);
}
template <>
void StrideASum<double>(const double* x, double* res, int n, int stride) {
res[0] = platform::dynload::cblas_dasum(n / stride, x, stride);
res[0] = phi::dynload::cblas_dasum(n / stride, x, stride);
}
// TODO(TJ): tuning me carefully on AVX, AVX2 and AVX512
......@@ -309,10 +308,9 @@ AWALYS_USE_ME_WITH_DOUBLE(Softmax);
} // namespace mkl
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
namespace mkl = paddle::operators::jit::more::mkl;
namespace mkl = phi::jit::more::mkl;
#define REGISTER_MKL_KERNEL(func) \
REGISTER_JITKERNEL_MORE( \
......
paddle/fluid/operators/jit/more/mkl/mkl.h → paddle/phi/kernels/funcs/jit/more/mkl/mkl.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -18,11 +18,10 @@
#include <type_traits>
#include <vector>
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace more {
namespace mkl {
......@@ -108,7 +107,7 @@ void EmbSeqPool(const T* table,
PADDLE_ENFORCE_EQ(
attr->table_width * attr->index_width,
attr->out_width,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute table_width * index_width of EmbSeqPool should "
"be equal to out_width. But table_width * index_width is %d, "
"out_width is %d.",
......@@ -118,19 +117,19 @@ void EmbSeqPool(const T* table,
PADDLE_ENFORCE_LT(
idx[i],
attr->table_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The idx shoud be lower than the attribute table_height of "
"EmbSeqPool. But %dth of idx is %d and table_height is %d.",
i,
idx[i],
attr->table_height));
PADDLE_ENFORCE_GE(idx[i],
0,
platform::errors::InvalidArgument(
"The idx shoud be equal to or larger than "
"the 0. But %dth of idx is %d.",
i,
idx[i]));
PADDLE_ENFORCE_GE(
idx[i],
0,
phi::errors::InvalidArgument("The idx shoud be equal to or larger than "
"the 0. But %dth of idx is %d.",
i,
idx[i]));
};
for (int64_t w = 0; w != attr->index_width; ++w) {
......@@ -200,7 +199,7 @@ void Sgd(const T* lr,
const sgd_attr_t* attr) {
PADDLE_ENFORCE_EQ(attr->param_width,
attr->grad_width,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute param_width of Sgd should be "
"equal to the attribute grad_width. But param_width "
"is %d and grad_width is %d.",
......@@ -208,7 +207,7 @@ void Sgd(const T* lr,
attr->grad_width));
PADDLE_ENFORCE_LE(attr->selected_rows_size,
attr->grad_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute selected_rows_size of Sgd should be "
"equal to or less than the attribute grad_height. "
"But selected_rows_size is %d and grad_height is %d.",
......@@ -221,7 +220,7 @@ void Sgd(const T* lr,
auto h_idx = rows[i];
PADDLE_ENFORCE_LT(h_idx,
attr->param_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The rows of Sgd should be "
"less than the attribute. But %dth of rows "
"is %d and grad_width is %d.",
......@@ -231,11 +230,11 @@ void Sgd(const T* lr,
PADDLE_ENFORCE_GE(
h_idx,
0,
platform::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
phi::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
VAXPY(scalar, grad + i * width, out + h_idx * width, width);
}
} else {
......@@ -243,7 +242,7 @@ void Sgd(const T* lr,
auto h_idx = rows[i];
PADDLE_ENFORCE_LT(h_idx,
attr->param_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The rows of Sgd should be "
"less than the attribute. But %dth of rows "
"is %d and grad_width is %d.",
......@@ -253,11 +252,11 @@ void Sgd(const T* lr,
PADDLE_ENFORCE_GE(
h_idx,
0,
platform::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
phi::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
VScal(&scalar, grad + i * width, out + h_idx * width, width);
VAdd(param + h_idx * width,
out + h_idx * width,
......@@ -306,5 +305,4 @@ DECLARE_MKL_KERNEL(VBroadcast);
} // namespace mkl
} // namespace more
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/refer/refer.cc → paddle/phi/kernels/funcs/jit/refer/refer.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License. */
#include "paddle/fluid/operators/jit/refer/refer.h"
#include "paddle/phi/kernels/funcs/jit/refer/refer.h"
#include "paddle/fluid/operators/jit/registry.h"
#include "paddle/phi/kernels/funcs/jit/registry.h"
namespace refer = paddle::operators::jit::refer;
namespace refer = phi::jit::refer;
#define REGISTER_REFER_KERNEL(func) \
REGISTER_JITKERNEL_REFER( \
......
paddle/fluid/operators/jit/refer/refer.h → paddle/phi/kernels/funcs/jit/refer/refer.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -18,12 +18,11 @@
#include <limits>
#include <string>
#include "paddle/fluid/operators/jit/helper.h"
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/helper.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
namespace refer {
......@@ -147,7 +146,7 @@ void (*getActFunc(KernelType type))(const T*, T*, int) { // NOLINT
} else if (type == kVIdentity) {
return VIdentity<T>;
}
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(phi::errors::Unimplemented(
"Act JIT kernel do not support type: %s.", type));
return nullptr;
}
......@@ -482,7 +481,7 @@ void EmbSeqPool(const T* table,
PADDLE_ENFORCE_EQ(
attr->table_width * attr->index_width,
attr->out_width,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute table_width * index_width of EmbSeqPool should "
"be equal to out_width. But table_width * index_width is %d and "
"out_width is %d.",
......@@ -493,19 +492,19 @@ void EmbSeqPool(const T* table,
PADDLE_ENFORCE_LT(
idx[i],
attr->table_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The idx shoud be lower than the attribute table_height of "
"EmbSeqPool. But %dth of idx is %d and table_height is %d.",
i,
idx[i],
attr->table_height));
PADDLE_ENFORCE_GE(idx[i],
0,
platform::errors::InvalidArgument(
"The idx shoud be equal to or larger than "
"the 0. But %dth of idx is %d.",
i,
idx[i]));
PADDLE_ENFORCE_GE(
idx[i],
0,
phi::errors::InvalidArgument("The idx shoud be equal to or larger than "
"the 0. But %dth of idx is %d.",
i,
idx[i]));
};
for (int64_t w = 0; w != attr->index_width; ++w) {
......@@ -549,7 +548,7 @@ void Sgd(const T* lr,
const sgd_attr_t* attr) {
PADDLE_ENFORCE_EQ(attr->param_width,
attr->grad_width,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute param_width of Sgd should be "
"equal to the attribute grad_width. But param_width "
"is %d and grad_width is %d.",
......@@ -557,7 +556,7 @@ void Sgd(const T* lr,
attr->grad_width));
PADDLE_ENFORCE_LE(attr->selected_rows_size,
attr->grad_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The attribute selected_rows_size of Sgd should be "
"equal to or less than the attribute grad_height. "
"But selected_rows_size is %d and grad_height is %d.",
......@@ -567,7 +566,7 @@ void Sgd(const T* lr,
auto h_idx = rows[i];
PADDLE_ENFORCE_LT(h_idx,
attr->param_height,
platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The rows of Sgd should be "
"less than the attribute. But %dth of rows "
"is %d and grad_width is %d.",
......@@ -577,11 +576,11 @@ void Sgd(const T* lr,
PADDLE_ENFORCE_GE(
h_idx,
0,
platform::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
phi::errors::InvalidArgument("The rows of Sgd should be "
"larger than 0. But %dth of rows "
"is %d.",
i,
h_idx));
for (int64_t j = 0; j < attr->grad_width; ++j) {
out[h_idx * attr->grad_width + j] =
param[h_idx * attr->grad_width + j] -
......@@ -698,5 +697,4 @@ DECLARE_REFER_KERNEL(VBroadcast);
} // namespace refer
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/registry.h → paddle/phi/kernels/funcs/jit/registry.h
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -19,13 +19,12 @@
#include <type_traits>
#include <utility> // for std::move
#include "paddle/fluid/operators/jit/kernel_base.h"
#include "paddle/fluid/operators/jit/kernel_pool.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/core/macros.h"
#include "paddle/phi/kernels/funcs/jit/kernel_base.h"
#include "paddle/phi/kernels/funcs/jit/kernel_pool.h"
namespace paddle {
namespace operators {
namespace phi {
namespace jit {
// make_unique is supported since c++14
......@@ -84,23 +83,22 @@ class JitKernelRegistrar {
msg)
// Refer always on CPUPlace
#define REGISTER_JITKERNEL_REFER(kernel_type, ...) \
STATIC_ASSERT_JITKERNEL_GLOBAL_NAMESPACE( \
__reg_jitkernel_##kernel_type##_refer_CPUPlace, \
"REGISTER_KERNEL_REFER must be called in global namespace"); \
static ::paddle::operators::jit::JitKernelRegistrar< \
::paddle::operators::jit::ReferKernelPool, \
::paddle::platform::CPUPlace, \
__VA_ARGS__> \
__jit_kernel_registrar_##kernel_type##_refer_CPUPlace_( \
::paddle::operators::jit::KernelType::kernel_type); \
int TouchJitKernelReg_##kernel_type##_refer_CPUPlace_() { \
__jit_kernel_registrar_##kernel_type##_refer_CPUPlace_.Touch(); \
return 0; \
#define REGISTER_JITKERNEL_REFER(kernel_type, ...) \
STATIC_ASSERT_JITKERNEL_GLOBAL_NAMESPACE( \
__reg_jitkernel_##kernel_type##_refer_CPUPlace, \
"REGISTER_KERNEL_REFER must be called in global namespace"); \
static ::phi::jit::JitKernelRegistrar<::phi::jit::ReferKernelPool, \
::phi::CPUPlace, \
__VA_ARGS__> \
__jit_kernel_registrar_##kernel_type##_refer_CPUPlace_( \
::phi::jit::KernelType::kernel_type); \
int TouchJitKernelReg_##kernel_type##_refer_CPUPlace_() { \
__jit_kernel_registrar_##kernel_type##_refer_CPUPlace_.Touch(); \
return 0; \
}
// kernel_type: should be in paddle::operators::jit::KernelType
// place_type: should be one of CPUPlace and GPUPlace in paddle::platform
// kernel_type: should be in phi::jit::KernelType
// place_type: should be one of CPUPlace and GPUPlace in phi
#define REGISTER_KERNEL_MORE(kernel_type, impl_type, place_type, ...) \
STATIC_ASSERT_JITKERNEL_GLOBAL_NAMESPACE( \
__reg_jitkernel_##kernel_type##_##impl_type##_##place_type, \
......@@ -108,12 +106,11 @@ class JitKernelRegistrar {
extern int TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static int __assert_##kernel_type##_##impl_type##_##place_type##_has_refer_ \
UNUSED = TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static ::paddle::operators::jit::JitKernelRegistrar< \
::paddle::operators::jit::KernelPool, \
::paddle::platform::place_type, \
__VA_ARGS__> \
static ::phi::jit::JitKernelRegistrar<::phi::jit::KernelPool, \
::phi::place_type, \
__VA_ARGS__> \
__jit_kernel_registrar_##kernel_type##_##impl_type##_##place_type##_( \
::paddle::operators::jit::KernelType::kernel_type); \
::phi::jit::KernelType::kernel_type); \
int TouchJitKernelReg_##kernel_type##_##impl_type##_##place_type##_() { \
__jit_kernel_registrar_##kernel_type##_##impl_type##_##place_type##_ \
.Touch(); \
......@@ -126,22 +123,21 @@ class JitKernelRegistrar {
#define REGISTER_GPUKERNEL_MORE(kernel_type, impl_type, ...) \
REGISTER_KERNEL_MORE(kernel_type, impl_type, GPUPlace, __VA_ARGS__)
#define REGISTER_JITKERNEL_GEN(kernel_type, ...) \
STATIC_ASSERT_JITKERNEL_GLOBAL_NAMESPACE( \
__reg_jitkernel_gen_##kernel_type##_CPUPlace_, \
"REGISTER_JITKERNEL_GEN must be called in global namespace"); \
extern int TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static int __assert_gen_##kernel_type##_has_refer_ UNUSED = \
TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static ::paddle::operators::jit::JitKernelRegistrar< \
::paddle::operators::jit::JitCodeCreatorPool, \
::paddle::platform::CPUPlace, \
__VA_ARGS__> \
__jit_kernel_registrar_gen_##kernel_type##_CPUPlace_( \
::paddle::operators::jit::KernelType::kernel_type); \
int TouchJitKernelReg_gen_##kernel_type##_CPUPlace_() { \
__jit_kernel_registrar_gen_##kernel_type##_CPUPlace_.Touch(); \
return 0; \
#define REGISTER_JITKERNEL_GEN(kernel_type, ...) \
STATIC_ASSERT_JITKERNEL_GLOBAL_NAMESPACE( \
__reg_jitkernel_gen_##kernel_type##_CPUPlace_, \
"REGISTER_JITKERNEL_GEN must be called in global namespace"); \
extern int TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static int __assert_gen_##kernel_type##_has_refer_ UNUSED = \
TouchJitKernelReg_##kernel_type##_refer_CPUPlace_(); \
static ::phi::jit::JitKernelRegistrar<::phi::jit::JitCodeCreatorPool, \
::phi::CPUPlace, \
__VA_ARGS__> \
__jit_kernel_registrar_gen_##kernel_type##_CPUPlace_( \
::phi::jit::KernelType::kernel_type); \
int TouchJitKernelReg_gen_##kernel_type##_CPUPlace_() { \
__jit_kernel_registrar_gen_##kernel_type##_CPUPlace_.Touch(); \
return 0; \
}
#define USE_JITKERNEL_GEN(kernel_type) \
......@@ -174,5 +170,4 @@ class JitKernelRegistrar {
USE_KERNEL_MORE(kernel_type, impl_type, CPUPlace)
} // namespace jit
} // namespace operators
} // namespace paddle
} // namespace phi
paddle/fluid/operators/jit/test.cc → paddle/phi/kernels/funcs/jit/test.cc
浏览文件 @ 2d36c9a9
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
......@@ -18,9 +18,10 @@ limitations under the License. */
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "gtest/gtest.h"
#include "paddle/fluid/operators/jit/kernels.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/common/place.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
DEFINE_double(acc, 1e-5, "Test accuracy threshold.");
......@@ -62,8 +63,8 @@ std::vector<int> TestSizes() {
return s;
}
namespace jit = paddle::operators::jit;
using CPUPlace = paddle::platform::CPUPlace;
namespace jit = phi::jit;
using CPUPlace = phi::CPUPlace;
template <typename KernelTuple,
typename PlaceType,
......@@ -1128,7 +1129,7 @@ void TestKernelSgd() {
const int64_t upper) -> std::vector<int64_t> {
PADDLE_ENFORCE_LE(static_cast<size_t>(upper - lower),
n - 1,
paddle::platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The range of Sgd (upper - lower) should be lower "
"than n-1 (Sgd size -1). But the upper - lower is %d "
"and n-1 is %d.",
......@@ -1137,7 +1138,7 @@ void TestKernelSgd() {
PADDLE_ENFORCE_GT(
n,
0,
paddle::platform::errors::InvalidArgument(
phi::errors::InvalidArgument(
"The Sgd size should be larger than 0. But the n is %d.", n));
std::vector<int64_t> all, out;
for (int i = 0; i < n; ++i) {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或