From ef5d216e2ad190ae6dc83f9e4293bfa36b6e0e8c Mon Sep 17 00:00:00 2001 From: zyfncg Date: Wed, 16 Feb 2022 11:07:13 +0800 Subject: [PATCH] change the format of api yaml (#39532) --- python/paddle/utils/code_gen/api.yaml | 42 +++++++++---------- python/paddle/utils/code_gen/api_base.py | 48 ++++++++++++---------- python/paddle/utils/code_gen/backward.yaml | 20 ++++----- 3 files changed, 58 insertions(+), 52 deletions(-) diff --git a/python/paddle/utils/code_gen/api.yaml b/python/paddle/utils/code_gen/api.yaml index 8b8b001739f..66411d00f15 100644 --- a/python/paddle/utils/code_gen/api.yaml +++ b/python/paddle/utils/code_gen/api.yaml @@ -1,5 +1,5 @@ - api : add - args : (const Tensor& x, const Tensor& y) + args : (Tensor x, Tensor y) output : Tensor infer_meta : func : ElementwiseInferMeta @@ -7,7 +7,7 @@ func : add - api : cast - args : (const Tensor& x, DataType out_dtype) + args : (Tensor x, DataType out_dtype) output : Tensor infer_meta : func : CastInferMeta @@ -18,7 +18,7 @@ - api : concat - args : (const std::vector& x, const Scalar& axis) + args : (Tensor[] x, Scalar axis) output : Tensor infer_meta : func : ConcatInferMeta @@ -27,7 +27,7 @@ func : concat - api : conj - args : (const Tensor& x) + args : (Tensor x) output : Tensor infer_meta : func : UnchangedInferMeta @@ -35,7 +35,7 @@ func : conj - api : divide - args : (const Tensor& x, const Tensor& y) + args : (Tensor x, Tensor y) output : Tensor infer_meta : func : ElementwiseInferMeta @@ -43,7 +43,7 @@ func : divide - api : dot - args : (const Tensor& x, const Tensor& y) + args : (Tensor x, Tensor y) output : Tensor infer_meta : func : DotInferMeta @@ -51,7 +51,7 @@ func : dot - api : empty - args : (const ScalarArray& shape, DataType dtype=DataType::FLOAT32, Backend place=Backend::CPU, DataLayout layout=DataLayout::NCHW) + args : (ScalarArray shape, DataType dtype=DataType::FLOAT32, Backend place=Backend::CPU, DataLayout layout=DataLayout::NCHW) output: Tensor infer_meta : func : CreateInferMeta @@ -64,7 +64,7 @@ layout : layout - api : empty_like - args : (const Tensor& x, DataType dtype = DataType::UNDEFINED, Backend place = Backend::UNDEFINED, DataLayout layout = DataLayout::UNDEFINED) + args : (Tensor x, DataType dtype = DataType::UNDEFINED, Backend place = Backend::UNDEFINED, DataLayout layout = DataLayout::UNDEFINED) output: Tensor infer_meta : func : CreateLikeInferMeta @@ -77,7 +77,7 @@ layout : layout > x - api : flatten - args : (const Tensor& x, int start_axis, int stop_axis) + args : (Tensor x, int start_axis, int stop_axis) output : Tensor infer_meta : func : FlattenInferMeta @@ -85,7 +85,7 @@ func : flatten - api : full - args : (const ScalarArray& shape, const Scalar& value, DataType dtype=DataType::FLOAT32, Backend place=Backend::CPU, DataLayout layout=DataLayout::NCHW) + args : (ScalarArray shape, Scalar value, DataType dtype=DataType::FLOAT32, Backend place=Backend::CPU, DataLayout layout=DataLayout::NCHW) output: Tensor infer_meta : func : CreateInferMeta @@ -98,7 +98,7 @@ layout : layout - api : full_like - args : (const Tensor& x, const Scalar& value, DataType dtype = DataType::UNDEFINED, Backend place = Backend::UNDEFINED, DataLayout layout = DataLayout::UNDEFINED) + args : (Tensor x, Scalar value, DataType dtype = DataType::UNDEFINED, Backend place = Backend::UNDEFINED, DataLayout layout = DataLayout::UNDEFINED) output: Tensor infer_meta : func : CreateLikeInferMeta @@ -111,7 +111,7 @@ layout : layout > x - api : matmul - args : (const Tensor& x, const Tensor& y, bool transpose_x = false, bool transpose_y = false) + args : (Tensor x, Tensor y, bool transpose_x = false, bool transpose_y = false) output : Tensor infer_meta : func : MatmulInferMeta @@ -120,7 +120,7 @@ backward : matmul_grad - api : mean - args : (const Tensor& x, const std::vector& axis={}, bool keep_dim=false) + args : (Tensor x, int64_t[] axis={}, bool keep_dim=false) output : Tensor infer_meta : func : ReduceInferMeta @@ -128,7 +128,7 @@ func : mean - api : multiply - args : (const Tensor& x, const Tensor& y) + args : (Tensor x, Tensor y) output : Tensor infer_meta : func : ElementwiseInferMeta @@ -136,12 +136,12 @@ func : multiply - api : ones_like - args : (const Tensor& x, DataType dtype=DataType::UNDEFINED, Backend place=Backend::UNDEFINED, DataLayout layout=DataLayout::UNDEFINED) + args : (Tensor x, DataType dtype=DataType::UNDEFINED, Backend place=Backend::UNDEFINED, DataLayout layout=DataLayout::UNDEFINED) output : Tensor invoke : full_like(x, 1, dtype, place, layout) - api : reshape - args : (const Tensor& x, const ScalarArray& shape) + args : (Tensor x, ScalarArray shape) output : Tensor infer_meta : func : ReshapeInferMeta @@ -149,7 +149,7 @@ func : reshape - api : scale - args : (const Tensor& x, const Scalar& scale, float bias, bool bias_after_scale) + args : (Tensor x, Scalar scale, float bias, bool bias_after_scale) output : Tensor infer_meta : func : UnchangedInferMeta @@ -158,7 +158,7 @@ func : scale, scale_sr - api : sign - args : (const Tensor& x) + args : (Tensor x) output : Tensor infer_meta : func : UnchangedInferMeta @@ -166,7 +166,7 @@ func : sign - api : subtract - args : (const Tensor& x, const Tensor& y) + args : (Tensor x, Tensor y) output : Tensor infer_meta : func : ElementwiseInferMeta @@ -174,7 +174,7 @@ func : subtract - api : sum - args : (const Tensor& x, const std::vector& axis={}, DataType dtype=DataType::UNDEFINED, bool keep_dim=false) + args : (Tensor x, int64_t[] axis={}, DataType dtype=DataType::UNDEFINED, bool keep_dim=false) output : Tensor infer_meta : func : SumInferMeta @@ -184,6 +184,6 @@ data_type : x - api : zeros_like - args : (const Tensor& x, DataType dtype=DataType::UNDEFINED, Backend place=Backend::UNDEFINED, DataLayout layout=DataLayout::UNDEFINED) + args : (Tensor x, DataType dtype=DataType::UNDEFINED, Backend place=Backend::UNDEFINED, DataLayout layout=DataLayout::UNDEFINED) output : Tensor invoke : full_like(x, 0, dtype, place, layout) diff --git a/python/paddle/utils/code_gen/api_base.py b/python/paddle/utils/code_gen/api_base.py index 2e1ed58e1c4..73c3ba4e4b4 100644 --- a/python/paddle/utils/code_gen/api_base.py +++ b/python/paddle/utils/code_gen/api_base.py @@ -71,23 +71,26 @@ class BaseAPI(object): f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml." args_str = args_str[1:-1] args_list = args_str.split(',') - input_types = [ - 'const Tensor&', 'const Tensor &', 'const std::vector&', - 'const std::vector &' - ] - attr_types = ['const Scalar&', 'const Scalar &', 'const ScalarArray&', 'const ScalarArray &', \ - 'int', 'int32_t', 'int64_t', 'size_t', 'float', 'double', 'bool', \ - 'const std::vector&', 'Backend', 'DataLayout', 'DataType'] + input_types_map = { + 'Tensor': 'const Tensor&', + 'Tensor[]': 'const std::vector&' + } + attr_types_map = {'ScalarArray' : 'const ScalarArray&', 'Scalar' : 'const Scalar&', \ + 'int' : 'int', 'int32_t' : 'int32_t', 'int64_t' : 'int64_t', 'size_t' : 'size_t', \ + 'float' : 'float', 'double' : 'double', 'bool' : 'bool', \ + 'Backend' : 'Backend', 'DataLayout' : 'DataLayout', 'DataType' : 'DataType', \ + 'int64_t[]' : 'const std::vector&', 'int[]' : 'const std::vector&'} args_declare_str = "" args_define_str = "" for item in args_list: item = item.strip() + type_and_name = item.split(' ') # match the input tensor has_input = False - for in_type in input_types: - if item.startswith(in_type): - input_name = item[len(in_type):].strip() + for in_type_symbol, in_type in input_types_map.items(): + if type_and_name[0] == in_type_symbol: + input_name = type_and_name[1].strip() assert len(input_name) > 0, \ f"The input tensor name should not be empty. Please check the args of {api_name} in yaml." assert len(attrs['names']) == 0, \ @@ -103,9 +106,9 @@ class BaseAPI(object): continue # match the attribute - for attr_type in attr_types: - if item.startswith(attr_type): - attr_name = item[len(attr_type):].strip() + for attr_type_symbol, attr_type in attr_types_map.items(): + if type_and_name[0] == attr_type_symbol: + attr_name = item[len(attr_type_symbol):].strip() assert len(attr_name) > 0, \ f"The attribute name should not be empty. Please check the args of {api_name} in yaml." default_value = None @@ -128,25 +131,28 @@ class BaseAPI(object): def parse_output(self, api_name, output_config): def parse_output_item(output_item): - alllowd_output_types = ['Tensor', 'std::vector'] + output_type_map = { + 'Tensor': 'Tensor', + 'Tensor[]': 'std::vector' + } if re.search(r'\(\w*\)', output_item): result = re.search( - r"(?P[a-zA-Z0-9_<>]+)\s*\((?P\w+)\)", + r"(?P[a-zA-Z0-9_[\]]+)\s*\((?P\w+)\)", output_item) out_type = result.group('out_type') - assert out_type in alllowd_output_types, \ - f"{api_name} : Output type error: the output type only support Tensor and std::vector, \ + assert out_type in output_type_map, \ + f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \ but now is {out_type}." return out_type, result.group('name') else: - if output_item.strip() in alllowd_output_types: - return output_item.strip(), 'out' + if output_item.strip() in output_type_map: + return output_type_map[output_item.strip()], 'out' else: raise ValueError( - "{} : Output type error: the output type only support Tensor and std::vector, \ - but now is {}.".format(api_name, out_type)) + "{} : Output type error: the output type only support Tensor and Tensor[], \ + but now is {}.".format(api_name, output_item.strip())) temp_list = output_config.split(',') diff --git a/python/paddle/utils/code_gen/backward.yaml b/python/paddle/utils/code_gen/backward.yaml index b1227217b3c..62b724432e9 100644 --- a/python/paddle/utils/code_gen/backward.yaml +++ b/python/paddle/utils/code_gen/backward.yaml @@ -1,6 +1,6 @@ - backward_api : matmul_grad - forward : matmul (const Tensor& x, const Tensor& y, bool transpose_x=false, bool transpose_y=false) -> Tensor(out) - args : (const Tensor& x, const Tensor& y, const Tensor& out_grad, bool transpose_x=false, bool transpose_y=false) + forward : matmul (Tensor x, Tensor y, bool transpose_x=false, bool transpose_y=false) -> Tensor(out) + args : (Tensor x, Tensor y, Tensor out_grad, bool transpose_x=false, bool transpose_y=false) output : Tensor(x_grad), Tensor(y_grad) infer_meta : func : GeneralBinaryGradInferMeta @@ -9,26 +9,26 @@ func : matmul_grad - backward_api : scale_grad - forward : scale (const Tensor& x, const Scalar& scale, float bias, bool bias_after_scale) -> Tensor(out) - args : (const Tensor& out_grad, const Scalar& scale, float bias=0.0, bool bias_after_scale=true) + forward : scale (Tensor x, Scalar scale, float bias, bool bias_after_scale) -> Tensor(out) + args : (Tensor out_grad, Scalar scale, float bias=0.0, bool bias_after_scale=true) output : Tensor(x_grad) invoke : scale(out_grad, scale, bias, bias_after_scale) # TODO(zhangyunfei) The config of double grad and triple grad will be supported in the future. # # - backward_api : matmul_double_grad -# forward : matmul_grad (const Tensor& x, const Tensor& y, const Tensor& out_grad, bool transpose_x, bool transpose_y) -> tuple(dx, dy) -# args : (const Tensor& x, const Tensor& y, const Tensor& out_grad, const Tensor& dx_grad, const Tensor& dy_grad, bool transpose_x, bool transpose_y) -# output : tuple // d2x, d2y, dout_grad +# forward : matmul_grad (Tensor x, Tensor y, Tensor out_grad, bool transpose_x, bool transpose_y) -> Tensor(dx), Tensor>(dy) +# args : (Tensor x, Tensor y, Tensor out_grad, Tensor dx_grad, Tensor dy_grad, bool transpose_x, bool transpose_y) +# output : Tensor(d2x), Tensor(d2y), Tensor(dout_grad) # infer_meta : # func : MatmulDoubleGradInferMeta # kernel : # func : matmul_double_grad # - backward_api : matmul_triple_grad -# forward : matmul_double_grad (const Tensor& x, const Tensor& y, const Tensor& out_grad, const Tensor& dx_grad, const Tensor& dy_grad, bool transpose_x, bool transpose_y) -> tuple(d2x, d2y, dout_grad) -# args : (const Tensor& x, const Tensor& y, const Tensor& out_grad, const Tensor& dx_grad, const Tensor& dy_grad, const Tensor& d2x_grad, const Tensor& d2y_grad, const Tensor& dout_grad_grad, bool transpose_x, bool transpose_y) -# output : tuple // d3x, d3y, d2out_grad, ddx_grad, ddy_grad +# forward : matmul_double_grad (Tensor x, Tensor y, Tensor out_grad, Tensor dx_grad, Tensor dy_grad, bool transpose_x, bool transpose_y) -> Tensor(d2x), Tensor(d2y), Tensor(dout_grad) +# args : (Tensor x, Tensor y, Tensor out_grad, Tensor dx_grad, Tensor dy_grad, Tensor d2x_grad, Tensor d2y_grad, Tensor dout_grad_grad, bool transpose_x, bool transpose_y) +# output : Tensor(d3x), Tensor(d3y), Tensor(d2out_grad), Tensor(ddx_grad), Tensor(ddy_grad) # infer_meta : # func : MatmulTripleGradInferMeta # kernel : -- GitLab