未验证 提交 5dfc0cd6 编写于 作者: C Chen Weihang 提交者: GitHub

[Phi] Move all yaml to phi subdirectory (#43986)

* move yaml to phi

* fix undef var error
上级 53d5abe3
......@@ -70,4 +70,4 @@ paddle/fluid/pybind/tmp_eager_final_state_op_function_impl.h
# these files (directories) are generated before build system generation
paddle/fluid/operators/generated_op.cc
paddle/phi/ops/compat/generated_sig.cc
python/paddle/utils/code_gen/parsed_apis/
paddle/phi/api/yaml/parsed_apis/
set(api_yaml_path
"${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/legacy_api.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_api.yaml"
"${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/api.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_api.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_api.yaml"
)
set(backward_yaml_path
"${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/backward.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/legacy_backward.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_bw_api.yaml"
"${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_bw_api.yaml"
)
set(tmp_forwards_cc_path
"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/eager_generated/forwards/tmp_dygraph_functions.cc"
......@@ -30,7 +30,7 @@ set(nodes_h_path
)
# StringTensor only needs forward api
set(fwd_api_yaml_path
"${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/strings_api.yaml")
"${PADDLE_SOURCE_DIR}/paddle/phi/api/yaml/strings_api.yaml")
message("Final State Eager CodeGen")
add_custom_target(
......
......@@ -17,13 +17,13 @@ else()
DEPS tensor_base dense_tensor phi_api_utils phi_enforce)
endif()
set(api_gen_base ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api_base.py)
set(api_gen_base ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/api_base.py)
# forward api file
set(api_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api_gen.py)
set(api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml)
set(api_gen_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/api_gen.py)
set(api_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/api.yaml)
set(legacy_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/legacy_api.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_api.yaml)
set(api_header_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/include/api.h)
set(api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/api.cc)
set(api_header_file_tmp ${api_header_file}.tmp)
......@@ -31,11 +31,10 @@ set(api_source_file_tmp ${api_source_file}.tmp)
# backward api file
set(bw_api_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/backward_api_gen.py)
set(bw_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/backward.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/backward_api_gen.py)
set(bw_api_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/backward.yaml)
set(legacy_bw_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/legacy_backward.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_backward.yaml)
set(bw_api_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/backward/backward_api.h)
set(bw_api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/backward_api.cc)
......@@ -44,7 +43,7 @@ set(bw_api_source_file_tmp ${bw_api_source_file}.tmp)
# dygraph(intermediate) api file
set(im_api_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/intermediate_api_gen.py)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/intermediate_api_gen.py)
set(dygraph_api_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/dygraph_api.h)
set(dygraph_api_source_file
......@@ -54,9 +53,9 @@ set(dygraph_api_source_file_tmp ${dygraph_api_source_file}.tmp)
# sparse api file
set(sparse_api_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_api_gen.py)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/sparse_api_gen.py)
set(sparse_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_api.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_api.yaml)
set(sparse_api_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/include/sparse_api.h)
set(sparse_api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/sparse_api.cc)
......@@ -65,9 +64,9 @@ set(sparse_api_source_file_tmp ${sparse_api_source_file}.tmp)
# sparse bw api file
set(sparse_bw_api_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_bw_api_gen.py)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py)
set(sparse_bw_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_bw_api.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_bw_api.yaml)
set(sparse_bw_api_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/backward/sparse_bw_api.h)
set(sparse_bw_api_source_file
......@@ -77,9 +76,9 @@ set(sparse_bw_api_source_file_tmp ${sparse_bw_api_source_file}.tmp)
# strings api file
set(strings_api_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/strings_api_gen.py)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/strings_api_gen.py)
set(strings_api_yaml_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/strings_api.yaml)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/strings_api.yaml)
set(strings_api_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/include/strings_api.h)
set(strings_api_source_file
......@@ -89,7 +88,7 @@ set(strings_api_source_file_tmp ${strings_api_source_file}.tmp)
# wrapped infermeta file
set(wrapped_infermeta_gen_file
${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/wrapped_infermeta_gen.py)
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py)
set(wrapped_infermeta_header_file
${CMAKE_SOURCE_DIR}/paddle/phi/infermeta/generated.h)
set(wrapped_infermeta_source_file
......@@ -109,7 +108,7 @@ else()
endif()
# parse apis
set(parsed_api_dir ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/parsed_apis)
set(parsed_api_dir ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/parsed_apis)
set(generated_op_path
${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_op.cc)
set(generated_argument_mapping_path
......@@ -121,18 +120,20 @@ message(
- ${bw_api_yaml_file}
- ${legacy_bw_api_yaml_file}")
execute_process(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml
COMMAND ${CMAKE_COMMAND} -E make_directory ${parsed_api_dir}
COMMAND ${PYTHON_EXECUTABLE} parse_api.py --api_yaml_path ./api.yaml
COMMAND ${PYTHON_EXECUTABLE} generator/parse_api.py --api_yaml_path ./api.yaml
--output_path ./parsed_apis/api.parsed.yaml
COMMAND ${PYTHON_EXECUTABLE} parse_api.py --api_yaml_path ./legacy_api.yaml
--output_path ./parsed_apis/legacy_api.parsed.yaml
COMMAND ${PYTHON_EXECUTABLE} parse_api.py --api_yaml_path ./backward.yaml
--output_path ./parsed_apis/backward_api.parsed.yaml --backward
COMMAND ${PYTHON_EXECUTABLE} generator/parse_api.py --api_yaml_path
./legacy_api.yaml --output_path ./parsed_apis/legacy_api.parsed.yaml
COMMAND
${PYTHON_EXECUTABLE} parse_api.py --api_yaml_path ./legacy_backward.yaml
--output_path ./parsed_apis/legacy_backward_api.parsed.yaml --backward
RESULTS_VARIABLE _results)
${PYTHON_EXECUTABLE} generator/parse_api.py --api_yaml_path ./backward.yaml
--output_path ./parsed_apis/backward_api.parsed.yaml --backward
COMMAND
${PYTHON_EXECUTABLE} generator/parse_api.py --api_yaml_path
./legacy_backward.yaml --output_path
./parsed_apis/legacy_backward_api.parsed.yaml --backward RESULTS_VARIABLE
_results)
foreach(_result in ${_results})
if(${_result})
message(FATAL_ERROR "api yaml parsing failed, exiting.")
......@@ -144,9 +145,9 @@ message("validate api yaml:
- ${parsed_api_dir}/api.parsed.yaml
- ${parsed_api_dir}/backward_api.parsed.yaml")
execute_process(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml
COMMAND
${PYTHON_EXECUTABLE} cross_validate.py --forward_yaml_paths
${PYTHON_EXECUTABLE} generator/cross_validate.py --forward_yaml_paths
./parsed_apis/api.parsed.yaml ./parsed_apis/legacy_api.parsed.yaml
--backward_yaml_paths ./parsed_apis/backward_api.parsed.yaml
./parsed_apis/legacy_backward_api.parsed.yaml
......@@ -161,13 +162,13 @@ message(
create or remove auto-geneated argument mappings: ${generated_argument_mapping_path}.tmp"
)
execute_process(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml
COMMAND
${PYTHON_EXECUTABLE} generate_op.py --api_yaml_path
${PYTHON_EXECUTABLE} generator/generate_op.py --api_yaml_path
./parsed_apis/api.parsed.yaml --backward_api_yaml_path
./parsed_apis/backward_api.parsed.yaml --api_version_yaml_path
api_version.yaml --api_args_compat_yaml_path args_compat.yaml
--output_op_path "${generated_op_path}.tmp" --output_arg_map_path
api_version.yaml --api_compat_yaml_path api_compat.yaml --output_op_path
"${generated_op_path}.tmp" --output_arg_map_path
"${generated_argument_mapping_path}.tmp"
RESULT_VARIABLE _result)
if(${_result})
......
......@@ -154,7 +154,7 @@ class ForwardAPI(BaseAPI):
0] == 'dense' else 'SetSelectedRowsKernelOutput'
if return_type == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'][0] is not None, \
f"{api_name}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
{code_indent} auto kernel_out = {set_out_func}({self.outputs['out_size_expr'][0]}, kernel_backend, &api_output);"""
......@@ -199,7 +199,7 @@ class ForwardAPI(BaseAPI):
if out_dtype_list[i] == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'][i] is not None, \
f"{api_name}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
{code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['out_size_expr'][i]}, kernel_backend, {get_out_code});"""
......@@ -313,7 +313,7 @@ def main():
parser.add_argument('--api_yaml_path',
help='path to api yaml file',
nargs='+',
default='python/paddle/utils/code_gen/api.yaml')
default='paddle/phi/api/yaml/api.yaml')
parser.add_argument('--api_header_path',
help='output of generated api header code file',
......
......@@ -133,7 +133,7 @@ class BackwardAPI(BaseAPI):
0] == 'dense' else 'SetSelectedRowsKernelOutput'
if out_dtype_list[0] == 'std::vector<Tensor>':
assert self.outputs['out_size_expr'] is not None, \
f"{api_name}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
{code_indent} auto kernel_out = {set_out_func}(&{self.outputs['names'][0]});"""
......@@ -164,7 +164,7 @@ class BackwardAPI(BaseAPI):
{code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};"""
assert self.outputs['out_size_expr'][i] is not None, \
f"{api_name}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api."
output_create = output_create + f"""
{code_indent} auto kernel_out_{i} = {set_out_func}(&{self.outputs['names'][i]});"""
......@@ -279,7 +279,7 @@ def main():
parser.add_argument('--backward_yaml_path',
help='path to backward yaml file',
nargs='+',
default='python/paddle/utils/code_gen/backward.yaml')
default='paddle/phi/api/yaml/backward.yaml')
parser.add_argument('--backward_header_path',
help='output of generated backward header code file',
default='paddle/phi/api/backward/backward_api.h')
......
......@@ -54,7 +54,7 @@ def restruct_io(api):
return api
def main(api_yaml_path, backward_yaml_path, api_args_compat_yaml_path,
def main(api_yaml_path, backward_yaml_path, api_compat_yaml_path,
api_version_yaml_path, output_op_path, output_arg_map_path):
with open(api_yaml_path, "rt") as f:
apis = yaml.safe_load(f)
......@@ -72,7 +72,7 @@ def main(api_yaml_path, backward_yaml_path, api_args_compat_yaml_path,
for api_version in api_versions:
forward_api_dict[api_version['api']]['version'] = api_version['version']
with open(api_args_compat_yaml_path, "rt") as f:
with open(api_compat_yaml_path, "rt") as f:
api_args_map = yaml.safe_load(f)
# replace args name for OpMaker
for api_args in api_args_map:
......@@ -219,7 +219,7 @@ if __name__ == "__main__":
parser.add_argument('--backward_api_yaml_path',
type=str,
help="parsed backward api yaml file.")
parser.add_argument('--api_args_compat_yaml_path',
parser.add_argument('--api_compat_yaml_path',
type=str,
help="api args compat yaml file.")
parser.add_argument('--api_version_yaml_path',
......@@ -235,5 +235,5 @@ if __name__ == "__main__":
args = parser.parse_args()
main(args.api_yaml_path, args.backward_api_yaml_path,
args.api_args_compat_yaml_path, args.api_version_yaml_path,
args.api_compat_yaml_path, args.api_version_yaml_path,
args.output_op_path, args.output_arg_map_path)
......@@ -134,11 +134,11 @@ def main():
parser.add_argument('--api_yaml_path',
nargs='+',
help='path to api yaml file',
default='python/paddle/utils/code_gen/api.yaml')
default='paddle/phi/api/yaml/api.yaml')
parser.add_argument('--sparse_api_yaml_path',
help='path to sparse api yaml file',
default='python/paddle/utils/code_gen/sparse_api.yaml')
default='paddle/phi/api/yaml/sparse_api.yaml')
parser.add_argument('--dygraph_api_header_path',
help='output of generated dygraph api header code file',
......
......@@ -280,7 +280,7 @@ def main():
description='Generate PaddlePaddle C++ Sparse API files')
parser.add_argument('--api_yaml_path',
help='path to sparse api yaml file',
default='python/paddle/utils/code_gen/sparse_api.yaml')
default='paddle/phi/api/yaml/sparse_api.yaml')
parser.add_argument('--api_header_path',
help='output of generated api header code file',
......
......@@ -162,10 +162,9 @@ def generate_api(api_yaml_path, header_file_path, source_file_path):
def main():
parser = argparse.ArgumentParser(
description='Generate PaddlePaddle C++ Sparse API files')
parser.add_argument(
'--api_yaml_path',
help='path to sparse api yaml file',
default='python/paddle/utils/code_gen/sparse_bw_api.yaml')
parser.add_argument('--api_yaml_path',
help='path to sparse api yaml file',
default='paddle/phi/api/yaml/sparse_bw_api.yaml')
parser.add_argument('--api_header_path',
help='output of generated api header code file',
......
......@@ -351,7 +351,7 @@ def main():
description='Generate PaddlePaddle C++ Strings API files')
parser.add_argument('--api_yaml_path',
help='path to sparse api yaml file',
default='python/paddle/utils/code_gen/strings_api.yaml')
default='paddle/phi/api/yaml/strings_api.yaml')
parser.add_argument('--api_header_path',
help='output of generated api header code file',
......
{% from "operator_utils.c.j2" import name_map, register_name_map %}
// this file is generated by python/paddle/utils/code_gen/generate_op.py, do not edit.
// this file is generated by paddle/phi/api/yaml/generator/generate_op.py, do not edit.
#include "paddle/phi/core/compat/op_utils.h"
#include "paddle/utils/small_vector.h"
......
{% from "operator_utils.c.j2" import op_maker, backward_op_maker, operator, register_op_with_components, register_op_version %}
// this file is generated by python/paddle/utils/code_gen/generate_op.py, do not edit.
// this file is generated by paddle/phi/api/yaml/generator/generate_op.py, do not edit.
#include <string>
#include "paddle/fluid/framework/infershape_utils.h"
#include "paddle/fluid/framework/op_registry.h"
......
......@@ -160,7 +160,7 @@ def main():
parser.add_argument('--api_yaml_path',
help='path to api yaml file',
nargs='+',
default='python/paddle/utils/code_gen/api.yaml')
default='paddle/phi/api/yaml/api.yaml')
parser.add_argument(
'--wrapped_infermeta_header_path',
help='output of generated wrapped_infermeta header code file',
......
......@@ -72,12 +72,11 @@ def get_skipped_kernel_list():
def get_api_yaml_info(file_path):
apis = []
with open(file_path + "/python/paddle/utils/code_gen/api.yaml", 'r') as f:
with open(file_path + "/paddle/phi/api/yaml/api.yaml", 'r') as f:
api_list = yaml.load(f, Loader=yaml.FullLoader)
if api_list:
apis.extend(api_list)
with open(file_path + "/python/paddle/utils/code_gen/legacy_api.yaml",
'r') as f:
with open(file_path + "/paddle/phi/api/yaml/legacy_api.yaml", 'r') as f:
legacy_api_list = yaml.load(f, Loader=yaml.FullLoader)
if legacy_api_list:
apis.extend(legacy_api_list)
......
......@@ -77,8 +77,8 @@ done
#step 2:get simple general inferMeta function wrap info
temp_path=`mktemp -d`
python3 ${PADDLE_ROOT}/python/paddle/utils/code_gen/wrapped_infermeta_gen.py \
--api_yaml_path ${PADDLE_ROOT}/python/paddle/utils/code_gen/api.yaml ${PADDLE_ROOT}/python/paddle/utils/code_gen/legacy_api.yaml \
python3 ${PADDLE_ROOT}/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py \
--api_yaml_path ${PADDLE_ROOT}/paddle/phi/api/yaml/api.yaml ${PADDLE_ROOT}/paddle/phi/api/yaml/legacy_api.yaml \
--wrapped_infermeta_header_path ${temp_path}/generate.h \
--wrapped_infermeta_source_path ${temp_path}/generate.cc
......
......@@ -20,8 +20,8 @@ import yaml
from typing import List, Dict, Any
skipped_phi_api_list_file = "/tools/infrt/skipped_phi_api.json"
api_yaml_file = "/python/paddle/utils/code_gen/api.yaml"
legacy_api_yaml_file = "/python/paddle/utils/code_gen/legacy_api.yaml"
api_yaml_file = "/paddle/phi/api/yaml/api.yaml"
legacy_api_yaml_file = "/paddle/phi/api/yaml/legacy_api.yaml"
def get_skipped_kernel_list():
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册