add static_ops.yaml for static op (#48991)

.gitignore

@@ -73,8 +73,8 @@ tools/nvcc_lazy
 # these files (directories) are generated before build system generation
 paddle/fluid/operators/generated_op.cc
 paddle/fluid/operators/generated_sparse_op.cc
-paddle/phi/ops/compat/generated_sig.cc
-paddle/phi/ops/compat/generated_sparse_sig.cc
+paddle/fluid/operators/generated_static_op.cc
+paddle/phi/ops/compat/generated_*.cc
 paddle/phi/api/yaml/parsed_apis/
 paddle/fluid/operators/generator/parsed_ops/
 paddle/fluid/pybind/tmp_eager_op_function_impl.h

paddle/fluid/operators/generator/CMakeLists.txt

@@ -5,6 +5,7 @@ include(phi)
 set(op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/ops.yaml)
 set(legacy_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_ops.yaml)
 set(bw_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/backward.yaml)
+set(static_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/static_ops.yaml)
 set(legacy_bw_op_yaml_file
     ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/legacy_backward.yaml)
 set(sparse_op_yaml_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/sparse_ops.yaml)
@@ -29,10 +30,14 @@ set(parsed_op_dir
     ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator/parsed_ops)
 set(generated_op_path
     ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_op.cc)
+set(generated_static_op_path
+    ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_static_op.cc)
 set(generated_sparse_ops_path
     ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_sparse_op.cc)
 set(generated_argument_mapping_path
     ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sig.cc)
+set(generated_static_argument_mapping_path
+    ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_static_sig.cc)
 set(generated_sparse_argument_mapping_path
     ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sparse_sig.cc)
 
@@ -54,6 +59,8 @@ execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${legacy_bw_op_yaml_file}
     --output_path ./parsed_ops/legacy_backward_ops.parsed.yaml --backward
+  COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${static_op_yaml_file}
+          --output_path ./parsed_ops/static_ops.parsed.yaml
   COMMAND ${PYTHON_EXECUTABLE} parse_op.py --op_yaml_path ${sparse_op_yaml_file}
           --output_path ./parsed_ops/sparse_ops.parsed.yaml
   COMMAND
@@ -75,7 +82,8 @@ execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} cross_validate.py --forward_yaml_paths
     ./parsed_ops/ops.parsed.yaml ./parsed_ops/legacy_ops.parsed.yaml
-    --backward_yaml_paths ./parsed_ops/backward_ops.parsed.yaml
+    ./parsed_ops/static_ops.parsed.yaml --backward_yaml_paths
+    ./parsed_ops/backward_ops.parsed.yaml
     ./parsed_ops/legacy_backward_ops.parsed.yaml
   RESULT_VARIABLE _result)
 if(${_result})
@@ -113,6 +121,20 @@ if(${_result})
   message(FATAL_ERROR "operator codegen failed, exiting.")
 endif()
 
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator
+  COMMAND
+    ${PYTHON_EXECUTABLE} generate_static_op.py --ops_yaml_path
+    ./parsed_ops/static_ops.parsed.yaml --op_version_yaml_path
+    ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_version.yaml
+    --op_compat_yaml_path ${CMAKE_SOURCE_DIR}/paddle/phi/api/yaml/op_compat.yaml
+    --output_op_path "${generated_static_op_path}.tmp" --output_arg_map_path
+    "${generated_static_argument_mapping_path}.tmp"
+  RESULT_VARIABLE _result)
+if(${_result})
+  message(FATAL_ERROR "operator codegen failed, exiting.")
+endif()
+
 execute_process(
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generator
   COMMAND
@@ -126,84 +148,33 @@ if(${_result})
   message(FATAL_ERROR "sparse operator codegen failed, exiting.")
 endif()
 
-if(EXISTS "${generated_op_path}.tmp" AND EXISTS "${generated_op_path}")
-  execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-                          "${generated_op_path}.tmp" "${generated_op_path}")
-  message("copy if different ${generated_op_path}.tmp ${generated_op_path}")
-elseif(EXISTS "${generated_op_path}.tmp")
-  execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_op_path}.tmp"
-                          "${generated_op_path}")
-  message("copy ${generated_op_path}.tmp ${generated_op_path}")
-else()
-  execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f "${generated_op_path}")
-  message("remove ${generated_op_path}")
-endif()
-
-if(EXISTS "${generated_sparse_ops_path}.tmp" AND EXISTS
-                                                 "${generated_sparse_ops_path}")
-  execute_process(
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-            "${generated_sparse_ops_path}.tmp" "${generated_sparse_ops_path}")
-  message(
-    "copy if different ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}"
-  )
-elseif(EXISTS "${generated_sparse_ops_path}.tmp")
-  execute_process(
-    COMMAND ${CMAKE_COMMAND} -E copy "${generated_sparse_ops_path}.tmp"
-            "${generated_sparse_ops_path}")
-  message("copy ${generated_sparse_ops_path}.tmp ${generated_sparse_ops_path}")
-else()
-  execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
-                          "${generated_sparse_ops_path}")
-  message("remove ${generated_sparse_ops_path}")
-endif()
-
-if(EXISTS "${generated_argument_mapping_path}.tmp"
-   AND EXISTS "${generated_argument_mapping_path}")
-  execute_process(
-    COMMAND
-      ${CMAKE_COMMAND} -E copy_if_different
-      "${generated_argument_mapping_path}.tmp"
-      "${generated_argument_mapping_path}")
-  message(
-    "copy if different ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}"
-  )
-elseif(EXISTS "${generated_argument_mapping_path}.tmp")
-  execute_process(
-    COMMAND ${CMAKE_COMMAND} -E copy "${generated_argument_mapping_path}.tmp"
-            "${generated_argument_mapping_path}")
-  message(
-    "copy ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}"
-  )
-else()
-  execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
-                          "${generated_argument_mapping_path}")
-  message("remove ${generated_argument_mapping_path}")
-endif()
-
-if(EXISTS "${generated_sparse_argument_mapping_path}.tmp"
-   AND EXISTS "${generated_sparse_argument_mapping_path}")
-  execute_process(
-    COMMAND
-      ${CMAKE_COMMAND} -E copy_if_different
-      "${generated_sparse_argument_mapping_path}.tmp"
-      "${generated_sparse_argument_mapping_path}")
-  message(
-    "copy if different ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}"
-  )
-elseif(EXISTS "${generated_sparse_argument_mapping_path}.tmp")
-  execute_process(
-    COMMAND
-      ${CMAKE_COMMAND} -E copy "${generated_sparse_argument_mapping_path}.tmp"
-      "${generated_sparse_argument_mapping_path}")
-  message(
-    "copy ${generated_sparse_argument_mapping_path}.tmp ${generated_sparse_argument_mapping_path}"
-  )
-else()
-  execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
-                          "${generated_sparse_argument_mapping_path}")
-  message("remove ${generated_sparse_argument_mapping_path}")
-endif()
+set(generated_static_files
+    "${generated_op_path}"
+    "${generated_static_op_path}"
+    "${generated_sparse_ops_path}"
+    "${generated_argument_mapping_path}"
+    "${generated_static_argument_mapping_path}"
+    "${generated_sparse_argument_mapping_path}")
+
+foreach(generated_static_file ${generated_static_files})
+  if(EXISTS "${generated_static_file}.tmp" AND EXISTS
+                                               "${generated_static_file}")
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different
+              "${generated_static_file}.tmp" "${generated_static_file}")
+    message(
+      "copy if different ${generated_static_file}.tmp ${generated_static_file}")
+  elseif(EXISTS "${generated_static_file}.tmp")
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -E copy "${generated_static_file}.tmp"
+              "${generated_static_file}")
+    message("copy ${generated_static_file}.tmp ${generated_static_file}")
+  else()
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f
+                            "${generated_static_file}")
+    message("remove ${generated_static_file}")
+  endif()
+endforeach()
 
 # op extra info file
 set(ops_extra_info_gen_file

paddle/fluid/operators/generator/generate_static_op.py

+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+from pathlib import Path
+
+import yaml
+from filters import (
+    cartesian_prod_mapping,
+    to_input_name,
+    to_int_array_tensor_name,
+    to_int_array_tensors_name,
+    to_op_attr_type,
+    to_opmaker_name,
+    to_opmaker_name_cstr,
+    to_pascal_case,
+    to_scalar_tensor_name,
+)
+from generate_op import replace_compat_name
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from parse_utils import to_named_dict
+from tests import (
+    is_base_op,
+    is_initializer_list,
+    is_scalar,
+    is_vec,
+    supports_inplace,
+    supports_no_need_buffer,
+)
+
+file_loader = FileSystemLoader(Path(__file__).parent / "templates")
+env = Environment(
+    loader=file_loader,
+    keep_trailing_newline=True,
+    trim_blocks=True,
+    lstrip_blocks=True,
+    undefined=StrictUndefined,
+    extensions=['jinja2.ext.do'],
+)
+env.filters["to_op_attr_type"] = to_op_attr_type
+env.filters["to_opmaker_name"] = to_opmaker_name
+env.filters["to_pascal_case"] = to_pascal_case
+env.filters["to_scalar_tensor_name"] = to_scalar_tensor_name
+env.filters["to_int_array_tensor_name"] = to_int_array_tensor_name
+env.filters["to_int_array_tensors_name"] = to_int_array_tensors_name
+env.filters["to_input_name"] = to_input_name
+env.filters["to_opmaker_name_cstr"] = to_opmaker_name_cstr
+env.filters["cartesian_prod_mapping"] = cartesian_prod_mapping
+env.tests["base_op"] = is_base_op
+env.tests["vec"] = is_vec
+env.tests["scalar"] = is_scalar
+env.tests["initializer_list"] = is_initializer_list
+env.tests["supports_inplace"] = supports_inplace
+env.tests["supports_no_need_buffer"] = supports_no_need_buffer
+
+
+def restruct_io(op):
+    op["input_dict"] = to_named_dict(op["inputs"])
+    op["attr_dict"] = to_named_dict(op["attrs"])
+    op["output_dict"] = to_named_dict(op["outputs"])
+    return op
+
+
+def main(
+    ops_yaml_path,
+    op_compat_yaml_path,
+    op_version_yaml_path,
+    output_op_path,
+    output_arg_map_path,
+):
+    with open(ops_yaml_path, "rt") as f:
+        ops = yaml.safe_load(f)
+        ops = [restruct_io(op) for op in ops]
+    forward_op_dict = to_named_dict(ops)
+
+    with open(op_version_yaml_path, "rt") as f:
+        op_versions = yaml.safe_load(f)
+
+    # add op version info into op
+    for op_version in op_versions:
+        if op_version['op'] in forward_op_dict:
+            forward_op_dict[op_version['op']]['version'] = op_version['version']
+
+    with open(op_compat_yaml_path, "rt") as f:
+        op_op_map = yaml.safe_load(f)
+
+    for op in ops:
+        op['op_name'] = op['name']
+
+    replace_compat_name(op_op_map, forward_op_dict, {})
+
+    if len(ops) == 0:
+        if os.path.isfile(output_op_path):
+            os.remove(output_op_path)
+        if os.path.isfile(output_arg_map_path):
+            os.remove(output_arg_map_path)
+        return
+
+    op_template = env.get_template('op.c.j2')
+    with open(output_op_path, "wt") as f:
+        msg = op_template.render(
+            ops=ops, backward_ops=[], op_dict=forward_op_dict
+        )
+        f.write(msg)
+
+    ks_template = env.get_template('ks.c.j2')
+    with open(output_arg_map_path, 'wt') as f:
+        msg = ks_template.render(ops=ops, backward_ops=[])
+        f.write(msg)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate operator file from op yaml."
+    )
+    parser.add_argument(
+        '--ops_yaml_path', type=str, help="parsed static ops yaml file."
+    )
+    parser.add_argument(
+        '--op_compat_yaml_path', type=str, help="ops args compat yaml file."
+    )
+    parser.add_argument(
+        '--op_version_yaml_path', type=str, help="ops version yaml file."
+    )
+    parser.add_argument(
+        "--output_op_path", type=str, help="path to save generated operators."
+    )
+    parser.add_argument(
+        "--output_arg_map_path",
+        type=str,
+        help="path to save generated argument mapping functions.",
+    )
+
+    args = parser.parse_args()
+    main(
+        args.ops_yaml_path,
+        args.op_compat_yaml_path,
+        args.op_version_yaml_path,
+        args.output_op_path,
+        args.output_arg_map_path,
+    )

paddle/phi/api/yaml/ops.yaml

@@ -1031,11 +1031,3 @@
   kernel :
     func : unfold
   backward : unfold_grad
-
-- op: share_buffer
-  args : (Tensor[] x, bool[] share_dims_and_dtype={})
-  output : Tensor[](out){x.size()}, Tensor[](xout){x.size()}
-  infer_meta :
-    func : ShareBufferInferMeta
-  kernel :
-    func : share_buffer

paddle/phi/api/yaml/static_ops.yaml

+- op : share_buffer
+  args : (Tensor[] x, bool[] share_dims_and_dtype={})
+  output : Tensor[](out){x.size()}, Tensor[](xout){x.size()}
+  infer_meta :
+    func : ShareBufferInferMeta
+  kernel :
+    func : share_buffer