未验证 提交 8f87f0c7 编写于 作者: Z zyfncg 提交者: GitHub

add static_ops.yaml for static op (#48991)

上级 93e36b06
......@@ -73,8 +73,8 @@ tools/nvcc_lazy
# these files (directories) are generated before build system generation
paddle/fluid/operators/generated_op.cc
paddle/fluid/operators/generated_sparse_op.cc
paddle/phi/ops/compat/generated_sig.cc
paddle/phi/ops/compat/generated_sparse_sig.cc
paddle/fluid/operators/generated_static_op.cc
paddle/phi/ops/compat/generated_*.cc
paddle/phi/api/yaml/parsed_apis/
paddle/fluid/operators/generator/parsed_ops/
paddle/fluid/pybind/tmp_eager_op_function_impl.h
......
......@@ -5,6 +5,7 @@ include(phi)
set(op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/ops.yaml)
set(legacy_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_ops.yaml)
set(bw_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/backward.yaml)
set(static_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/static_ops.yaml)
set(legacy_bw_op_yaml_file
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_backward.yaml)
set(sparse_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_ops.yaml)
......@@ -29,10 +30,14 @@ set(parsed_op_dir
${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator/parsed_ops)
set(generated_op_path
${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_op.cc)
set(generated_static_op_path
${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_static_op.cc)
set(generated_sparse_ops_path
${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_sparse_op.cc)
set(generated_argument_mapping_path
${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sig.cc)
set(generated_static_argument_mapping_path
${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_static_sig.cc)
set(generated_sparse_argument_mapping_path
${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sparse_sig.cc)
......@@ -54,6 +59,8 @@ execute_process(
COMMAND
${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${legacy_bw_op_yaml_file}
--output_path ./parsed_ops/legacy_backward_ops.parsed.yaml --backward
COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${static_op_yaml_file}
--output_path ./parsed_ops/static_ops.parsed.yaml
COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${sparse_op_yaml_file}
--output_path ./parsed_ops/sparse_ops.parsed.yaml
COMMAND
......@@ -75,7 +82,8 @@ execute_process(
COMMAND
${PYTHON_EXECUTABLE} cross_validate.py --forward_yaml_paths
./parsed_ops/ops.parsed.yaml ./parsed_ops/legacy_ops.parsed.yaml
--backward_yaml_paths ./parsed_ops/backward_ops.parsed.yaml
./parsed_ops/static_ops.parsed.yaml --backward_yaml_paths
./parsed_ops/backward_ops.parsed.yaml
./parsed_ops/legacy_backward_ops.parsed.yaml
RESULT_VARIABLE _result)
if(${_result})
......@@ -113,6 +121,20 @@ if(${_result})
message(FATAL_ERROR "operator codegen failed, exiting.")
endif()
execute_process(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator
COMMAND
${PYTHON_EXECUTABLE} generate_static_op.py --ops_yaml_path
./parsed_ops/static_ops.parsed.yaml --op_version_yaml_path
${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_version.yaml
--op_compat_yaml_path ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_compat.yaml
--output_op_path "${generated_static_op_path}.tmp" --output_arg_map_path
"${generated_static_argument_mapping_path}.tmp"
RESULT_VARIABLE _result)
if(${_result})
message(FATAL_ERROR "operator codegen failed, exiting.")
endif()
execute_process(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator
COMMAND
......@@ -126,84 +148,33 @@ if(${_result})
message(FATAL_ERROR "sparse operator codegen failed, exiting.")
endif()
if(EXISTS "${generated_op_path}.tmp" AND EXISTS "${generated_op_path}")
execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${generated_op_path}.tmp" "${generated_op_path}")
message("copy if different ${generated_op_path}.tmp ${generated_op_path}")
elseif(EXISTS "${generated_op_path}.tmp")
execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_op_path}.tmp"
"${generated_op_path}")
message("copy ${generated_op_path}.tmp ${generated_op_path}")
else()
execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f "${generated_op_path}")
message("remove ${generated_op_path}")
endif()
if(EXISTS "${generated_sparse_ops_path}.tmp" AND EXISTS
"${generated_sparse_ops_path}")
execute_process(
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${generated_sparse_ops_path}.tmp" "${generated_sparse_ops_path}")
message(
"copy if different ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}"
)
elseif(EXISTS "${generated_sparse_ops_path}.tmp")
execute_process(
COMMAND ${CMAKE_COMMAND} -E copy "${generated_sparse_ops_path}.tmp"
"${generated_sparse_ops_path}")
message("copy ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}")
else()
execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
"${generated_sparse_ops_path}")
message("remove ${generated_sparse_ops_path}")
endif()
if(EXISTS "${generated_argument_mapping_path}.tmp"
AND EXISTS "${generated_argument_mapping_path}")
execute_process(
COMMAND
${CMAKE_COMMAND} -E copy_if_different
"${generated_argument_mapping_path}.tmp"
"${generated_argument_mapping_path}")
message(
"copy if different ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}"
)
elseif(EXISTS "${generated_argument_mapping_path}.tmp")
execute_process(
COMMAND ${CMAKE_COMMAND} -E copy "${generated_argument_mapping_path}.tmp"
"${generated_argument_mapping_path}")
message(
"copy ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}"
)
else()
execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
"${generated_argument_mapping_path}")
message("remove ${generated_argument_mapping_path}")
endif()
if(EXISTS "${generated_sparse_argument_mapping_path}.tmp"
AND EXISTS "${generated_sparse_argument_mapping_path}")
execute_process(
COMMAND
${CMAKE_COMMAND} -E copy_if_different
"${generated_sparse_argument_mapping_path}.tmp"
"${generated_sparse_argument_mapping_path}")
message(
"copy if different ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}"
)
elseif(EXISTS "${generated_sparse_argument_mapping_path}.tmp")
execute_process(
COMMAND
${CMAKE_COMMAND} -E copy "${generated_sparse_argument_mapping_path}.tmp"
"${generated_sparse_argument_mapping_path}")
message(
"copy ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}"
)
else()
execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
"${generated_sparse_argument_mapping_path}")
message("remove ${generated_sparse_argument_mapping_path}")
endif()
set(generated_static_files
"${generated_op_path}"
"${generated_static_op_path}"
"${generated_sparse_ops_path}"
"${generated_argument_mapping_path}"
"${generated_static_argument_mapping_path}"
"${generated_sparse_argument_mapping_path}")
foreach(generated_static_file ${generated_static_files})
if(EXISTS "${generated_static_file}.tmp" AND EXISTS
"${generated_static_file}")
execute_process(
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${generated_static_file}.tmp" "${generated_static_file}")
message(
"copy if different ${generated_static_file}.tmp ${generated_static_file}")
elseif(EXISTS "${generated_static_file}.tmp")
execute_process(
COMMAND ${CMAKE_COMMAND} -E copy "${generated_static_file}.tmp"
"${generated_static_file}")
message("copy ${generated_static_file}.tmp ${generated_static_file}")
else()
execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
"${generated_static_file}")
message("remove ${generated_static_file}")
endif()
endforeach()
# op extra info file
set(ops_extra_info_gen_file
......
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os
from pathlib import Path
import yaml
from filters import (
cartesian_prod_mapping,
to_input_name,
to_int_array_tensor_name,
to_int_array_tensors_name,
to_op_attr_type,
to_opmaker_name,
to_opmaker_name_cstr,
to_pascal_case,
to_scalar_tensor_name,
)
from generate_op import replace_compat_name
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from parse_utils import to_named_dict
from tests import (
is_base_op,
is_initializer_list,
is_scalar,
is_vec,
supports_inplace,
supports_no_need_buffer,
)
file_loader = FileSystemLoader(Path(__file__).parent / "templates")
env = Environment(
loader=file_loader,
keep_trailing_newline=True,
trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined,
extensions=['jinja2.ext.do'],
)
env.filters["to_op_attr_type"] = to_op_attr_type
env.filters["to_opmaker_name"] = to_opmaker_name
env.filters["to_pascal_case"] = to_pascal_case
env.filters["to_scalar_tensor_name"] = to_scalar_tensor_name
env.filters["to_int_array_tensor_name"] = to_int_array_tensor_name
env.filters["to_int_array_tensors_name"] = to_int_array_tensors_name
env.filters["to_input_name"] = to_input_name
env.filters["to_opmaker_name_cstr"] = to_opmaker_name_cstr
env.filters["cartesian_prod_mapping"] = cartesian_prod_mapping
env.tests["base_op"] = is_base_op
env.tests["vec"] = is_vec
env.tests["scalar"] = is_scalar
env.tests["initializer_list"] = is_initializer_list
env.tests["supports_inplace"] = supports_inplace
env.tests["supports_no_need_buffer"] = supports_no_need_buffer
def restruct_io(op):
op["input_dict"] = to_named_dict(op["inputs"])
op["attr_dict"] = to_named_dict(op["attrs"])
op["output_dict"] = to_named_dict(op["outputs"])
return op
def main(
ops_yaml_path,
op_compat_yaml_path,
op_version_yaml_path,
output_op_path,
output_arg_map_path,
):
with open(ops_yaml_path, "rt") as f:
ops = yaml.safe_load(f)
ops = [restruct_io(op) for op in ops]
forward_op_dict = to_named_dict(ops)
with open(op_version_yaml_path, "rt") as f:
op_versions = yaml.safe_load(f)
# add op version info into op
for op_version in op_versions:
if op_version['op'] in forward_op_dict:
forward_op_dict[op_version['op']]['version'] = op_version['version']
with open(op_compat_yaml_path, "rt") as f:
op_op_map = yaml.safe_load(f)
for op in ops:
op['op_name'] = op['name']
replace_compat_name(op_op_map, forward_op_dict, {})
if len(ops) == 0:
if os.path.isfile(output_op_path):
os.remove(output_op_path)
if os.path.isfile(output_arg_map_path):
os.remove(output_arg_map_path)
return
op_template = env.get_template('op.c.j2')
with open(output_op_path, "wt") as f:
msg = op_template.render(
ops=ops, backward_ops=[], op_dict=forward_op_dict
)
f.write(msg)
ks_template = env.get_template('ks.c.j2')
with open(output_arg_map_path, 'wt') as f:
msg = ks_template.render(ops=ops, backward_ops=[])
f.write(msg)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Generate operator file from op yaml."
)
parser.add_argument(
'--ops_yaml_path', type=str, help="parsed static ops yaml file."
)
parser.add_argument(
'--op_compat_yaml_path', type=str, help="ops args compat yaml file."
)
parser.add_argument(
'--op_version_yaml_path', type=str, help="ops version yaml file."
)
parser.add_argument(
"--output_op_path", type=str, help="path to save generated operators."
)
parser.add_argument(
"--output_arg_map_path",
type=str,
help="path to save generated argument mapping functions.",
)
args = parser.parse_args()
main(
args.ops_yaml_path,
args.op_compat_yaml_path,
args.op_version_yaml_path,
args.output_op_path,
args.output_arg_map_path,
)
......@@ -1031,11 +1031,3 @@
kernel :
func : unfold
backward : unfold_grad
- op: share_buffer
args : (Tensor[] x, bool[] share_dims_and_dtype={})
output : Tensor[](out){x.size()}, Tensor[](xout){x.size()}
infer_meta :
func : ShareBufferInferMeta
kernel :
func : share_buffer
- op : share_buffer
args : (Tensor[] x, bool[] share_dims_and_dtype={})
output : Tensor[](out){x.size()}, Tensor[](xout){x.size()}
infer_meta :
func : ShareBufferInferMeta
kernel :
func : share_buffer
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册