eager_gen.py

# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import argparse
import os
from codegen_utils import (
    core_ops_returns_info,
    core_ops_args_info,
    core_ops_args_type_info,
)
from codegen_utils import ReadBwdFile
from codegen_utils import FindForwardName, GetGradNodeName, GetSavedName
from codegen_utils import IsPlainTensorType, IsVectorTensorType
from codegen_utils import GetConstReference, RemoveConstAndReference
from codegen_utils import (
    GetDygraphForwardFunctionName,
    GetIntermediateAPIFunctionName,
)
from codegen_utils import GetAutoGradMetaName, GetAutoGradMetaVectorName
from codegen_utils import GetInplacedFunctionName
from codegen_utils import ParseYamlForwardFromBackward
from codegen_utils import ParseYamlBackward
from codegen_utils import ParseYamlInplaceInfo
from codegen_utils import FunctionGeneratorBase, GeneratorBase
from codegen_utils import ops_to_fill_zero_for_empty_grads
from codegen_utils import AssertMessage, GetIndent

# Note: assign is a inplace api when parameter(output) isn't none,
# so we should check parameter(output) with rule of inplace.
# But because there is no check in old dygraph mode, in order to
# keeping the code compatible, here we also skip inplace check in new dygraph temporarily,
# and this will be fixed in the futrue.
inplace_check_blacklist = set(["assign_out_"])

# Black Ops list that's NO NEED to apply code generation
black_ops_list = [
    "conv2d",
    "conv2d_grad",
    "conv2d_grad_grad",
    "add_n",
    "add_n_grad",
]


###########
## Utils ##
###########
def ParseArguments():
    parser = argparse.ArgumentParser(
        description='Eager Code Generator Args Parser'
    )
    parser.add_argument('--nodes_h_path', type=str)
    parser.add_argument('--nodes_cc_path', type=str)
    parser.add_argument('--forwards_h_path', type=str)
    parser.add_argument('--forwards_cc_path', type=str)
    parser.add_argument('--api_yaml_path', type=str)
    parser.add_argument('--backward_yaml_path', type=str)

    args = parser.parse_args()
    return args


########################
## Code Gen Templates ##
########################
SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = """  void SetTensorWrapper{}(const paddle::experimental::Tensor& {}) {{
    {} = egr::TensorWrapper({}, {});
  }}
"""

SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = """  void SetTensorWrapper{}(const std::vector<paddle::experimental::Tensor>& {}) {{
    for(const auto& eager_tensor : {}) {{
      {}.emplace_back(egr::TensorWrapper(eager_tensor, {}));
    }};
  }}
"""

PLAIN_TENSOR_MEMBER_TEMPLATE = """  egr::TensorWrapper {};
"""

VECTOR_TENSOR_MEMBER_TEMPLATE = """  std::vector<egr::TensorWrapper> {};
"""

CLEAR_TENSOR_WRAPPER_TEMPLATE = """    {}.clear();
"""

CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE = """    for (auto& tw : {}) {{
      tw.clear();
    }}
"""

SET_ATTR_METHOD_TEMPLATE = """  void SetAttribute{}({} {}) {{
    {} = {};
  }}
"""

ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE = """  {} {} = {};
"""

ATTRIBUTE_MEMBER_TEMPLATE = """  {} {};
"""

NODE_DECLARATION_TEMPLATE = """
class {} : public egr::GradNodeBase {{
 public:
  {}() : egr::GradNodeBase() {{}}
  {}(size_t bwd_in_slot_num, size_t bwd_out_slot_num) :
      egr::GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) {{}}
  ~{}() override = default;

  virtual paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize> operator()(
      paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize>& grads, bool create_graph = false, bool is_new_grad = false) override;
  std::string name() override {{ return \"{}\"; }}

  void ClearTensorWrappers() override {{
{}
    SetIsTensorWrappersCleared(true);
  }}

  std::shared_ptr<GradNodeBase> Copy() const override {{
    auto copied_node = std::shared_ptr<{}>(new {}(*this));
    return copied_node;
  }}

  // SetTensorWrapperX, SetTensorWrapperY, ...
{}
  // SetAttributes
{}
 private:
  // TensorWrappers
{}
  // Attributes
{}}};
"""

GRAD_FUNCTION_TEMPLATE = """
paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize> {}::operator()(paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize>& grads, bool create_graph, bool is_new_grad) {{
  VLOG(3) << \"Running AD API GRAD: \" << \"{}\";
  // Fill Zero For GradIn Tensors
{}
  // Apply Gradient Hooks
  auto hooked_grads = ApplyGradientHooks(grads);

  // Collect GradIn Tensors, Attrs and Recovered TensorWrappers
{}
  // Prepare Grad function call
{}
  // Runtime check if we need next grad
{}
  // Inplace Check
{}
  // Inplace Strategy
{}

  VLOG(5) << \"Running C++ API: \" << \"{}\";
  // Before log info
{}
  // Call grad_api function
{}
  // Check NaN and Inf id needed
{}
  // Get GradOut autograd_meta
{}
  // Create Grad Node
{}
  VLOG(4) << \"Finish AD API GRAD: {}";
  // LOG IF DEBUG
  {}
  // Return
{}
}}
"""

FORWARD_FUNCTION_TEMPLATE = """
{} {}({}) {{
  VLOG(3) << \"Running AD API: \" << \"{}\";
  // Dygraph Record Event
{}
  // AMP Logic
{}
  // Layout autotune
{}
  // Get Input AutoGradMeta
{}

  VLOG(5) << \"Running C++ API: \" << \"{}\";
 // Before log info
{}
 // Forward API Call
{}
  // Check NaN and Inf if needed
{}
  // Get Outputs
{}
  // Get Output AutoGradMeta
{}
  bool trace_backward = egr::Controller::Instance().HasGrad();
  bool require_any_grad = egr::EagerUtils::ComputeRequireGrad({});

  // Check Inplace if needed
{}{}
  // Node Creation
{}

  VLOG(4) << \"Finish AD API: {}";
  // LOG IF DEBUG
  {}
  // Returns
  return {};
}}
"""

AFTER_LOG_PRINT_TEMPLATE = """
  if(VLOG_IS_ON(4)){{
      const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s],  Output: [%s] }} \";
      {}
      VLOG(4) << paddle::string::Sprintf(INPUT_PRINT_TEMPLATE, input_str, output_str);
  }}
"""

BEFORE_LOG_PRINT_TEMPLATE = """
  if(VLOG_IS_ON(3)){{
      const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s]}} \";
      {}
      VLOG(3) << paddle::string::Sprintf(INPUT_PRINT_TEMPLATE, input_str);
  }}
"""

FORWARD_ONLY_FUNCTION_TEMPLATE = """
{} {}({}) {{
  VLOG(3) << \"Running AD API: \" << \"{}\";
  // Dygraph Record Event
{}
  // AMP Logic
{}
  // Layout autotune
{}
  VLOG(5) << \"Running C++ API: \" << \"{}\";
  // Before log info
{}
  // Forward API Call
{}
  // Get Outputs
{}
  VLOG(4) << \"Finish AD API: {}";
  // LOG IF DEBUG
  {}
  // Returns
  return {};
}}
"""

FORWARD_BODY_TEMPLATE = """  if(require_any_grad) {{
{}
    egr::EagerUtils::PassStopGradient({});

    // Node Construction
{}
    // SetAttributes if needed
{}
    // Set TensorWrappers for Forward Inputs if needed
{}
    // SetGradOutMeta & SetEdges
{}
    // SetOutRank & SetHistory & SetGradInMeta & RetainGrad
{}
{}
{}
{}
    // Set TensorWrappers for Forward Outputs if needed
{}
  }}
"""

HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = """  if(trace_backward) {{
{}
    // Node Construction
{}
    // SetAttributes if needed
{}
    // Set TensorWrappers for Forward Inputs if needed
{}
    // SetGradOutMeta & SetEdges
{}
    // SetOutRank & SetHistory & SetGradInMeta & RetainGrad
{}
{}
{}
{}
    // Set TensorWrappers for Forward Outputs if needed
{}
  }}
"""

NAMESPACE_WRAPPER_TEMPLATE = """
namespace {} {{
    {}
}}
"""

NODE_CC_FILE_TEMPLATE = """
#include "glog/logging.h"
#include "paddle/phi/api/all.h"
#include "paddle/phi/api/backward/backward_api.h"
#include "paddle/phi/api/backward/sparse_bw_api.h"
#include "paddle/fluid/imperative/tracer.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/platform/profiler/event_tracing.h"
#include "paddle/fluid/eager/utils.h"
#include "paddle/fluid/eager/api/utils/global_utils.h"
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/nodes.h"
#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
#include "paddle/fluid/eager/to_static/run_program_op_node.h"
#include "paddle/fluid/eager/nan_inf_utils.h"
#include "paddle/phi/api/include/sparse_api.h"
#include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h"
DECLARE_bool(check_nan_inf);
{}
"""

NODE_H_FILE_TEMPLATE = """
#pragma once
#include "paddle/fluid/eager/tensor_wrapper.h"
#include "paddle/fluid/eager/grad_node_info.h"
#include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h"

{}
"""

FORWARD_CC_FILE_TEMPLATE = """
#include "paddle/phi/api/lib/dygraph_api.h"
#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/nodes.h"
#include "paddle/fluid/eager/eager_layout_auto_tune.h"
#include "paddle/phi/api/include/strings_api.h"
#include "paddle/phi/api/include/sparse_api.h"
#include "paddle/fluid/eager/api/utils/global_utils.h"
#include "paddle/fluid/platform/profiler/event_tracing.h"
#include "paddle/fluid/eager/amp_utils.h"
#include "paddle/fluid/eager/eager_amp_auto_cast.h"
#include "paddle/phi/backends/gpu/gpu_info.h"
#include "paddle/fluid/eager/nan_inf_utils.h"
#include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h"
DECLARE_bool(check_nan_inf);
{}
{}
"""

FORWARD_H_FILE_TEMPLATE = """
#pragma once
#include "glog/logging.h"
#include "paddle/fluid/eager/autograd_meta.h"
#include "paddle/phi/api/all.h"
#include "paddle/fluid/eager/utils.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/eager/to_static/run_program_op_func.h"
#include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h"

using CPUPlace = phi::CPUPlace;
{}
{}
"""

CORE_OPS_INFO_TEMPLATE = """
std::unordered_map<std::string, std::vector<std::string>> core_ops_args_info = {{
    {}
}};
std::unordered_map<std::string, std::vector<std::string>> core_ops_args_type_info = {{
    {}
}};
std::unordered_map<std::string, std::vector<std::string>> core_ops_returns_info = {{
    {}
}};

"""

CORE_OPS_DECLARATION_TEMPLATE = """
extern std::unordered_map<std::string, std::vector<std::string>> core_ops_args_info;
extern std::unordered_map<std::string, std::vector<std::string>> core_ops_args_type_info;
extern std::unordered_map<std::string, std::vector<std::string>> core_ops_returns_info;

"""

CHECK_INPLACE_TEMPLATE = """
  egr::EagerUtils::CheckInplace({}, {}, require_any_grad);
"""

BUMP_INPLACE_VERSION_TEMPLATE = """
  // Bump Inplace Version
  {}.bump_inplace_version();
  VLOG(3) << \"Tensor(\" << {}.name() << \") uses Inplace Strategy.\";
"""

AMP_LOGIC_TEMPLATE = """  if (egr::Controller::Instance().GetAMPLevel() != paddle::imperative::AmpLevel::O0) {{
    VLOG(5) << "Check and Prepare For AMP";
    {}
    paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize> amp_tensors_vector = {};
    {}
    {}
    {}
    {{
      paddle::imperative::AutoCastGuard guard(egr::Controller::Instance().GetCurrentTracer(), paddle::imperative::AmpLevel::O0);
      {}
    }}
  }}
"""
LAYOUT_LOGIC_TEMPLATE = """
  if (egr::Controller::Instance().UseLayoutAutoTune()) {{
    paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize> tensors_vector = {};
    {}
    {}
    VLOG(5) << "Check and Prepare For LAYOUT "<< op_name;
    paddle::imperative::LayoutAutotuneGuard guard(egr::Controller::Instance().GetCurrentTracer(), false);
    {}
    {}
    // Returns
    return {};
  }}
"""
CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = """
  paddle::optional<paddle::experimental::Tensor> {}_optional;
  if({}.initialized()) {}_optional = paddle::make_optional<paddle::experimental::Tensor>({});
"""

CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = """
  paddle::optional<paddle::experimental::Tensor> {}_optional;
  if( {}.impl() ) {}_optional = paddle::make_optional<paddle::experimental::Tensor>({});
"""

CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = """
  paddle::optional<std::vector<paddle::experimental::Tensor>> {}_optional;
  if( !{}.empty() ) {}_optional = paddle::make_optional<std::vector<paddle::experimental::Tensor>>({});
"""

CHECK_BACKWARD_INPLACE_TEMPLATE = """
  bool can_be_inplaced = false;
  if ({}.initialized()) {{
    VLOG(10) << {}.name() << "({}) use_count: " << {}.impl().use_count();
    if ({}.impl().use_count() == 1 || ({}.impl().use_count() == 2 && {}.impl().get() == {}.impl().get())) {{
      can_be_inplaced = true;
    }}
  }}"""

CHECK_NAN_AND_INF_TEMPLATE = """  if (FLAGS_check_nan_inf) {{ egr::CheckTensorHasNanOrInf("{}", {}); }}
"""

inplace_optional_out_type_map = {
    "Tensor": "paddle::optional<paddle::experimental::Tensor>&",
    "std::vector<Tensor>": "paddle::optional<std::vector<paddle::experimental::Tensor>>&",
}


def ExtractForwardApiNameFormInvoke(invoke_config):
    api_name = invoke_config.split('(')[0]
    if api_name[-1] == '_':
        api_name = api_name[:-1]
    return re.search(
        r"(?P<api_name>[a-zA-Z0-9_]+)(?P<intermediate>_intermediate)?", api_name
    ).group('api_name')


def IsInvokeForwardApi(api_contents, forward_api_name_list):
    return (
        'invoke' in api_contents
        and ExtractForwardApiNameFormInvoke(api_contents['invoke'])
        in forward_api_name_list
    )


#######################
## Generator Helpers ##
#######################
def GenerateCoreOpInfoDeclaration():
    return CORE_OPS_DECLARATION_TEMPLATE


def GenerateCoreOpInfoDefinition():

    op_args_info_list = []
    for op_name, arg_list in core_ops_args_info.items():
        arg_str = ",".join(["\"" + v + "\"" for v in arg_list])
        op_args_info = f"{{ \"{op_name}\", {{ {arg_str} }} }},"
        op_args_info_list.append(op_args_info)

    op_types_info_list = []
    for op_name, type_list in core_ops_args_type_info.items():
        type_str = ",".join(["\"" + v + "\"" for v in type_list])
        op_types_info = f"{{ \"{op_name}\", {{ {type_str} }} }},"
        op_types_info_list.append(op_types_info)

    op_returns_info_list = []
    for op_name, return_list in core_ops_returns_info.items():
        return_str = ",".join(["\"" + v + "\"" for v in return_list])
        return_types_info = f"{{ \"{op_name}\", {{ {return_str} }} }},"
        op_returns_info_list.append(return_types_info)

    op_args_info_str = "\n".join(op_args_info_list)
    op_types_info_str = "\n".join(op_types_info_list)
    op_returns_info_str = "\n".join(op_returns_info_list)

    core_ops_info_definition_str = CORE_OPS_INFO_TEMPLATE.format(
        op_args_info_str, op_types_info_str, op_returns_info_str
    )

    return core_ops_info_definition_str


#####################
## Generator Class ##
#####################
class DygraphFunctionGeneratorBase(FunctionGeneratorBase):
    def __init__(
        self,
        forward_api_contents,
        grad_api_contents,
        forward_apis_dict,
        namespace,
    ):
        self.forward_api_contents = forward_api_contents
        # Members from Parent:
        # self.namespace
        # self.forward_api_contents
        # self.forward_api_name
        # self.orig_forward_inputs_list
        # self.orig_forward_attrs_list
        # self.orig_forward_returns_list
        # self.forward_inputs_position_map
        # self.forward_outputs_position_map
        # self.optional_inputs
        # self.no_need_buffers
        # self.intermediate_outputs
        # self.forward_inplace_map
        FunctionGeneratorBase.__init__(self, forward_api_contents, namespace)

        self.forward_apis_dict = forward_apis_dict
        self.grad_api_contents = grad_api_contents

        # Raw Contents
        self.backward_forward_str = ""
        self.backward_api_name = ""

        self.forward_attrs_list = (
            []
        )  # [ [attr_name, attr_type, default_value, orig_position], ...]
        self.forward_inputs_list = (
            []
        )  # [ [arg_name, arg_type, orig_position], ...]
        self.forward_returns_list = (
            []
        )  # [ [ret_name, ret_type, orig_position], ...]

        self.backward_attrs_list = (
            []
        )  # [ [attr_name, attr_type, default_value, orig_position], ...]
        self.backward_inputs_list = (
            []
        )  # [ [arg_name, arg_type, orig_position], ...]
        self.backward_returns_list = (
            []
        )  # [ [ret_name, ret_type, orig_position], ...]

        # SlotNameMatched Backward Data
        self.backward_forward_inputs_map = (
            {}
        )  # { "name" : [type, is_fwd_input, orig_position] ...}
        self.backward_grad_inputs_map = (
            {}
        )  # { "name" : [type, fwd_position, orig_position] ...}
        self.backward_grad_outputs_map = (
            {}
        )  # { "name" : [type, fwd_position, orig_position] ...}

        self.backward_inplace_map = {}  # {name : name, ...}

    def ParseBackwardInplaceInfo(self):
        grad_api_contents = self.grad_api_contents
        if 'inplace' not in grad_api_contents.keys():
            return

        inplace_map_str = grad_api_contents['inplace']
        self.backward_inplace_map = ParseYamlInplaceInfo(inplace_map_str)

    def DygraphYamlValidationCheck(self):
        forward_api_contents = self.forward_api_contents
        grad_api_contents = self.grad_api_contents

        assert (
            'op' in forward_api_contents.keys()
        ), "Unable to find \"op\" in ops.yaml"
        assert (
            'args' in forward_api_contents.keys()
        ), "Unable to find \"args\" in ops.yaml"
        assert (
            'output' in forward_api_contents.keys()
        ), "Unable to find \"output\" in ops.yaml"

        if grad_api_contents is not None:
            assert (
                'backward' in forward_api_contents.keys()
            ), "Unable to find \"backward\" in ops.yaml"
            assert (
                'args' in grad_api_contents.keys()
            ), "Unable to find \"args\" in backward.yaml"
            assert (
                'output' in grad_api_contents.keys()
            ), "Unable to find \"output\" in backward.yaml"
            assert (
                'forward' in grad_api_contents.keys()
            ), "Unable to find \"forward\" in backward.yaml"

    def ForwardsValidationCheck(self):
        forward_inputs_list = self.forward_inputs_list
        forward_attrs_list = self.forward_attrs_list
        forward_returns_list = self.forward_returns_list

        orig_forward_inputs_list = self.orig_forward_inputs_list
        orig_forward_attrs_list = self.orig_forward_attrs_list
        orig_forward_returns_list = self.orig_forward_returns_list

        for i in range(len(forward_inputs_list)):
            forward_input_type = forward_inputs_list[i][1]
            forward_input_pos = forward_inputs_list[i][2]
            orig_input_type = orig_forward_inputs_list[i][1]
            orig_input_pos = orig_forward_inputs_list[i][2]

            assert forward_input_type == orig_input_type, AssertMessage(
                forward_input_type, orig_input_type
            )
            assert forward_input_pos == orig_input_pos, AssertMessage(
                forward_input_pos, orig_input_pos
            )

        for i in range(len(forward_attrs_list)):
            orig_attr_type = orig_forward_attrs_list[i][1]
            orig_attr_pos = orig_forward_attrs_list[i][3]
            forward_attr_type = forward_attrs_list[i][1]
            forward_attr_pos = forward_attrs_list[i][3]
            assert orig_attr_type == forward_attr_type, AssertMessage(
                orig_attr_type, forward_attr_type
            )
            assert orig_attr_pos == forward_attr_pos, AssertMessage(
                orig_attr_pos, forward_attr_pos
            )

        for i in range(len(forward_returns_list)):
            orig_return_type = orig_forward_returns_list[i][1]
            orig_return_pos = orig_forward_returns_list[i][2]
            forward_return_type = forward_returns_list[i][1]
            forward_return_pos = forward_returns_list[i][2]

            assert orig_return_type == forward_return_type, AssertMessage(
                orig_return_type, forward_return_type
            )
            assert orig_return_pos == forward_return_pos, AssertMessage(
                orig_return_pos, forward_return_pos
            )

        # Check Order: Inputs, Attributes
        max_input_position = -1
        for _, _, pos in forward_inputs_list:
            max_input_position = max(max_input_position, pos)

        for _, _, _, pos in forward_attrs_list:
            assert pos > max_input_position, AssertMessage(
                pos, max_input_position
            )

    def BackwardValidationCheck(self):
        backward_forward_inputs_map = self.backward_forward_inputs_map
        backward_grad_inputs_map = self.backward_grad_inputs_map
        backward_attrs_list = self.backward_attrs_list

        # Check Order: TensorWrappers, GradTensors, Attributes
        max_fwd_input_position = -1
        for _, (_, _, pos) in backward_forward_inputs_map.items():
            max_fwd_input_position = max(max_fwd_input_position, pos)

        max_grad_tensor_position = -1
        for _, (_, _, pos) in backward_grad_inputs_map.items():
            assert pos > max_fwd_input_position, AssertMessage(
                pos, max_grad_tensor_position
            )
            max_grad_tensor_position = max(max_grad_tensor_position, pos)

        max_attr_position = -1
        for _, _, _, pos in backward_attrs_list:
            assert pos > max_grad_tensor_position, AssertMessage(
                pos, max_grad_tensor_position
            )
            max_attr_position = max(max_attr_position, pos)

    def IntermediateValidationCheck(self):
        intermediate_outputs = self.intermediate_outputs
        forward_returns_list = self.forward_returns_list
        """
        Check whether intermediate_outputs are positioned
        at the very end of forward_returns_list
        """
        intermediate_positions = range(
            len(forward_returns_list) - len(intermediate_outputs),
            len(forward_returns_list),
        )
        for ret_name, _, pos in forward_returns_list:
            if ret_name in intermediate_outputs:
                assert pos in intermediate_positions, AssertMessage(
                    pos, intermediate_positions
                )

    def CollectBackwardInfo(self):
        forward_api_contents = self.forward_api_contents
        grad_api_contents = self.grad_api_contents

        self.backward_api_name = forward_api_contents['backward']
        self.backward_forward_str = grad_api_contents['forward']
        backward_args_str = grad_api_contents['args']
        backward_returns_str = grad_api_contents['output']

        (
            self.backward_inputs_list,
            self.backward_attrs_list,
            self.backward_returns_list,
        ) = ParseYamlBackward(backward_args_str, backward_returns_str)

    def CollectForwardInfoFromBackwardContents(self):

        backward_forward_str = self.backward_forward_str

        (
            self.forward_inputs_list,
            self.forward_attrs_list,
            self.forward_returns_list,
        ) = ParseYamlForwardFromBackward(backward_forward_str)

    def CollectForwardInfoFromYamlForward(self):
        (
            self.forward_inputs_list,
            self.forward_attrs_list,
            self.forward_returns_list,
        ) = ParseYamlForwardFromBackward(
            self.forward_api_contents['args']
            + " -> "
            + self.forward_api_contents['output']
        )

    def SlotNameMatching(self):
        backward_inputs_list = self.backward_inputs_list
        backward_returns_list = self.backward_returns_list
        forward_inputs_position_map = self.forward_inputs_position_map
        forward_outputs_position_map = self.forward_outputs_position_map

        for backward_input in backward_inputs_list:
            backward_input_name = backward_input[0]
            backward_input_type = backward_input[1]
            backward_input_pos = backward_input[2]

            backward_fwd_name = FindForwardName(backward_input_name)
            if backward_fwd_name:
                # Grad Input
                assert (
                    backward_fwd_name in forward_outputs_position_map.keys()
                ), AssertMessage(
                    backward_fwd_name, forward_outputs_position_map.keys()
                )
                matched_forward_output_type = forward_outputs_position_map[
                    backward_fwd_name
                ][0]
                matched_forward_output_pos = forward_outputs_position_map[
                    backward_fwd_name
                ][1]

                self.backward_grad_inputs_map[backward_input_name] = [
                    backward_input_type,
                    matched_forward_output_pos,
                    backward_input_pos,
                ]
            else:
                # TensorWrapper Input
                if backward_input_name in forward_inputs_position_map.keys():
                    tensor_wrapper_type = forward_inputs_position_map[
                        backward_input_name
                    ][0]
                    self.backward_forward_inputs_map[backward_input_name] = [
                        backward_input_type,
                        True,
                        backward_input_pos,
                    ]

                elif backward_input_name in forward_outputs_position_map.keys():
                    tensor_wrapper_type = forward_outputs_position_map[
                        backward_input_name
                    ][0]
                    self.backward_forward_inputs_map[backward_input_name] = [
                        backward_input_type,
                        False,
                        backward_input_pos,
                    ]
                else:
                    assert (
                        False
                    ), f"Cannot find {backward_input_name} in forward position map"

        for backward_output in backward_returns_list:
            backward_output_name = backward_output[0]
            backward_output_type = backward_output[1]
            backward_output_pos = backward_output[2]

            backward_fwd_name = FindForwardName(backward_output_name)
            assert (
                backward_fwd_name is not None
            ), f"Detected {backward_fwd_name} = None"
            assert (
                backward_fwd_name in forward_inputs_position_map.keys()
            ), AssertMessage(
                backward_fwd_name, forward_inputs_position_map.keys()
            )

            matched_forward_input_type = forward_inputs_position_map[
                backward_fwd_name
            ][0]
            matched_forward_input_pos = forward_inputs_position_map[
                backward_fwd_name
            ][1]

            self.backward_grad_outputs_map[backward_output_name] = [
                backward_output_type,
                matched_forward_input_pos,
                backward_output_pos,
            ]

    def GetPassStopGradientArgsList(self, forward_outputs_position_map):
        pass_stop_gradient_args_list = ["false"]
        for name, (_, _) in forward_outputs_position_map.items():
            output_autograd_meta_name = GetAutoGradMetaName(name)
            pass_stop_gradient_args_list.append(output_autograd_meta_name)
        pass_stop_gradient_args_str = ",".join(pass_stop_gradient_args_list)
        return pass_stop_gradient_args_str

    def GenerateNodeCreationCodes(self, for_backward=False):
        forward_api_name = self.forward_api_name
        forward_inputs_position_map = self.forward_inputs_position_map
        forward_outputs_position_map = self.forward_outputs_position_map
        forward_attrs_list = self.forward_attrs_list
        backward_forward_inputs_map = self.backward_forward_inputs_map
        backward_grad_inputs_map = self.backward_grad_inputs_map
        backward_grad_outputs_map = self.backward_grad_outputs_map
        backward_attrs_list = self.backward_attrs_list
        optional_inputs = self.optional_inputs

        # Pass Stop Gradient Args
        pass_stop_gradient_args_str = self.GetPassStopGradientArgsList(
            forward_outputs_position_map
        )

        # Node Construction
        num_backward_inputs = len(forward_outputs_position_map.keys())
        num_backward_outputs = len(forward_inputs_position_map.keys())
        grad_node_name = GetGradNodeName(self.backward_api_name)

        # Helper
        indent = GetIndent(2)
        # NOTE(Aurelius74): DO NOT use make_shared here. Because some Node contains experimental::Scalar
        # which contains "complex128" as data. "complex128" is memory-aligned manually. But make_shared
        # request MEMALIGN for allocation (Maybe).
        # See https://stackoverflow.com/questions/31228656/how-can-shared-ptr-disrupt-alignment
        # and https://github.com/MRtrix3/mrtrix3/issues/957
        node_construction_str = f"{indent}auto grad_node = std::shared_ptr<{grad_node_name}>(new {grad_node_name}({num_backward_inputs}, {num_backward_outputs}));"

        # SetAttributes
        set_attributes_list = []
        forward_attrs_name_set = set()
        for name, _, _, _ in forward_attrs_list:
            forward_attrs_name_set.add(name)

        for name, _, default_val_attr, _ in backward_attrs_list:
            if name in forward_attrs_name_set:
                set_attributes = (
                    f"{indent}grad_node->SetAttribute{name}({name});"
                )
            else:
                set_attributes = f"{indent}grad_node->SetAttribute{name}({default_val_attr});"
            set_attributes_list.append(set_attributes)
        set_attributes_str = "\n".join(set_attributes_list)

        # SetTensorWrappers
        set_input_tensor_wrappers_list = []
        set_output_tensor_wrappers_list = []
        num_fwd_outputs = len(forward_outputs_position_map.keys())
        for name, (
            atype,
            is_fwd_input,
            pos,
        ) in backward_forward_inputs_map.items():
            is_optional = name in optional_inputs

            if is_fwd_input:
                if is_optional:
                    set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});"
                else:
                    set_tensor_wrappers = (
                        f"{indent}grad_node->SetTensorWrapper{name}({name});"
                    )
                set_input_tensor_wrappers_list.append(set_tensor_wrappers)
            else:  # Forwad's output as backward's input
                if num_fwd_outputs > 1:
                    # Aligned with forward output position
                    assert (
                        name in forward_outputs_position_map.keys()
                    ), AssertMessage(name, forward_outputs_position_map.keys())

                if is_optional:
                    set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});"
                else:
                    set_tensor_wrappers = (
                        f"{indent}grad_node->SetTensorWrapper{name}({name});"
                    )
                set_output_tensor_wrappers_list.append(set_tensor_wrappers)
        set_input_tensor_wrappers_str = "\n".join(
            set_input_tensor_wrappers_list
        )
        set_output_tensor_wrappers_str = "\n".join(
            set_output_tensor_wrappers_list
        )

        # SetGradOutMeta & SetEdges
        grad_node_out_list = []
        set_grad_out_meta_list = []
        set_edges_list = []
        for name, (_, pos) in forward_inputs_position_map.items():
            # Has corresponding grad output
            has_corresponding_grad_output = False
            for _, (
                _,
                corresponding_pos,
                _,
            ) in backward_grad_outputs_map.items():
                if pos == corresponding_pos:
                    has_corresponding_grad_output = True
            if not has_corresponding_grad_output:
                continue

            grad_node_out_list.append(name)
            is_optional = name in self.optional_inputs
            if is_optional:
                set_grad_out_meta = f"{indent}if({name}.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}.get_ptr()), {pos});"
            else:
                set_grad_out_meta = (
                    f"{indent}grad_node->SetGradOutMeta({name}, {pos});"
                )

            set_grad_out_meta_list.append(set_grad_out_meta)
        set_grad_out_meta_str = "\n".join(set_grad_out_meta_list)

        # SetOutRank & SetHistory & SetGradInMeta
        set_out_rank_list = []
        set_history_list = []
        set_grad_in_meta_list = []
        set_retain_grad_list = []
        num_outputs = len(forward_outputs_position_map.keys())
        for name, (_, pos) in forward_outputs_position_map.items():
            output_autograd_meta_name = GetAutoGradMetaName(name)
            set_out_rank = f"""{indent}if ({output_autograd_meta_name}) {{
{indent}  egr::EagerUtils::SetOutRankWithSlot({output_autograd_meta_name}, {pos});
{indent}}}"""

            set_history = f"""{indent}if ({output_autograd_meta_name}) {{
{indent}  egr::EagerUtils::SetHistory({output_autograd_meta_name}, grad_node);
{indent}}}"""

            set_grad_in_meta = (
                f"{indent}grad_node->SetGradInMeta({name}, {pos});"
            )
            set_retain_grad = (
                f"{indent}egr::EagerUtils::CheckAndRetainGrad({name});"
            )

            set_out_rank_list.append(set_out_rank)
            set_history_list.append(set_history)
            set_grad_in_meta_list.append(set_grad_in_meta)
            set_retain_grad_list.append(set_retain_grad)

        set_out_rank_str = "\n".join(set_out_rank_list)
        set_history_str = "\n".join(set_history_list)
        set_grad_in_meta_str = "\n".join(set_grad_in_meta_list)
        set_retain_grad_str = "\n".join(set_retain_grad_list)

        node_event_name = forward_api_name + " node_creation"
        node_creation_event_str = f"{indent}paddle::platform::RecordEvent node_creation_record_event(\"{node_event_name}\", paddle::platform::TracerEventType::OperatorInner, 1);\n"
        if not for_backward:
            self.node_creation_str = FORWARD_BODY_TEMPLATE.format(
                node_creation_event_str,
                pass_stop_gradient_args_str,
                node_construction_str,
                set_attributes_str,
                set_input_tensor_wrappers_str,
                set_grad_out_meta_str,
                set_out_rank_str,
                set_history_str,
                set_grad_in_meta_str,
                set_retain_grad_str,
                set_output_tensor_wrappers_str,
            )
        else:
            self.node_creation_str = (
                HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format(
                    node_creation_event_str,
                    node_construction_str,
                    set_attributes_str,
                    set_input_tensor_wrappers_str,
                    set_grad_out_meta_str,
                    set_out_rank_str,
                    set_history_str,
                    set_grad_in_meta_str,
                    set_retain_grad_str,
                    set_output_tensor_wrappers_str,
                )
            )

        self.grad_node_out_list = grad_node_out_list

    def run(self):
        # Basic Validation Check
        self.DygraphYamlValidationCheck()

        ##########################
        ## Parsing Raw Contents ##
        ##########################
        # Parse forward and backward inplace_map
        self.ParseForwardInplaceInfo()
        if self.grad_api_contents is not None:
            self.ParseBackwardInplaceInfo()
            # Parse no_need_buffer
            self.ParseNoNeedBuffer()

        # Parse optional_inputs
        self.ParseDispensable()

        # Parse intermediate_outputs
        self.ParseIntermediate()
        self.IntermediateValidationCheck()

        if self.grad_api_contents is not None:
            # Initialize backward_forward_str, backward_inputs_list, backward_attrs_list, backward_returns_list
            self.CollectBackwardInfo()

            # Initialize forward_inputs_list, forward_attrs_list, forward_returns_list
            self.CollectForwardInfoFromBackwardContents()

        if self.is_forward_only:
            self.CollectForwardInfoFromYamlForward()

        # Initialize orig_forward_inputs_list, orig_forward_attrs_list, orig_forward_returns_list
        self.CollectOriginalForwardInfo()

        # Forwards Validation Check
        self.ForwardsValidationCheck()

        #############################
        ## Process Parsed Contents ##
        #############################
        # Initialize forward_inputs_position_map, forward_outputs_position_map
        self.DetermineForwardPositionMap(
            self.forward_inputs_list, self.forward_returns_list
        )

        if self.grad_api_contents is not None:
            # Initialize backward_forward_inputs_map, backward_grad_inputs_map, backward_grad_outputs_map
            self.SlotNameMatching()
            # Backward Validation Check
            self.BackwardValidationCheck()


class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase):
    def __init__(
        self,
        forward_api_contents,
        grad_api_contents,
        forward_apis_dict,
        namespace,
    ):
        DygraphFunctionGeneratorBase.__init__(
            self,
            forward_api_contents,
            grad_api_contents,
            forward_apis_dict,
            namespace,
        )

        # Generated Results
        self.forward_definition_str = ""
        self.forward_declaration_str = ""

    def GenerateForwardLayoutAutotune(
        self,
        forward_api_name,
        amp_tensors_vector_list,
        layout_tensors_vector_optional_list,
        layout_autotune_list_str,
        returns_type_str,
        returns_str,
        amp_inputs_call_args_str,
    ):
        intermediate_outputs = self.intermediate_outputs
        forward_attrs_list = self.forward_attrs_list
        forward_outputs_position_map = self.forward_outputs_position_map
        num_outputs = len(forward_outputs_position_map.keys()) - len(
            intermediate_outputs
        )
        # for layout autotune attr
        lightly_sensitive_attr = [
            'axis',
            'axes',
            'dim',
            'dims',
            'start',
            'end',
            'stop',
            'perm',
        ]
        heavily_sensitive_attr = ['data_format', 'data_layout']
        layout_autotune_attr = []
        layout_autotune_attr_code_list = []
        layout_autotune_attr_type_list = []
        layout_autotune_attr_code_list.append(
            f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");\n"
        )

        lightly_flag = False
        heavily_flag = False
        for name, atype, default_val, pos in forward_attrs_list:
            for attr_name in lightly_sensitive_attr:
                if name.find(attr_name) != -1 and (
                    name not in layout_autotune_attr
                ):
                    lightly_flag = True
                    layout_autotune_attr.append(name)
                    layout_autotune_attr_type_list.append(atype)
            if lightly_flag is False:
                for attr_name in heavily_sensitive_attr:
                    if name.find(attr_name) != -1 and (
                        name not in layout_autotune_attr
                    ):
                        layout_autotune_attr.append(name)
                        layout_autotune_attr_type_list.append(atype)
                        heavily_flag = True
        if len(layout_autotune_attr) == 0:
            layout_autotune_attr_code_list.append(
                "auto transformer = egr::EagerLayoutAutotune(op_name, tensors_vector);\n"
            )
        elif len(layout_autotune_attr) == 1:
            layout_autotune_attr_code_list.append(
                f"auto transformer = egr::EagerLayoutAutotune<{layout_autotune_attr_type_list[0]}>(op_name, tensors_vector, &{layout_autotune_attr[0]});\n"
            )
        elif len(layout_autotune_attr) == 2:
            layout_autotune_attr_code_list.append(
                f"auto transformer = egr::EagerLayoutAutotune<{layout_autotune_attr_type_list[0]}, {layout_autotune_attr_type_list[1]}>(op_name, tensors_vector, &{layout_autotune_attr[0]}, &{layout_autotune_attr[1]});\n"
            )
        else:
            layout_autotune_attr_code_list.append(
                f"auto transformer = egr::EagerLayoutAutotune<{layout_autotune_attr_type_list[0]}>(op_name, tensors_vector,&{layout_autotune_attr[0]});\n"
            )
        # Out tensor
        layout_inputs_call_args_str = amp_inputs_call_args_str
        forward_function_name = GetDygraphForwardFunctionName(forward_api_name)
        layout_tmp_result_list = []
        layout_autotune_outs_list = []
        result_name = "api_result"
        if num_outputs == 1:
            result_name = returns_str
            layout_autotune_outs_list.append(
                f"transformer -> SetOutTensorLayout(&{returns_str});\n"
            )
        else:
            for name, (rtype, pos) in forward_outputs_position_map.items():
                if name in intermediate_outputs:
                    continue
                layout_autotune_outs_list.append(
                    f"    auto& {name} = std::get<{len(layout_tmp_result_list)}>(api_result);\n"
                )
                layout_autotune_outs_list.append(
                    f"    transformer -> SetOutTensorLayout(&{name});\n"
                )
                layout_tmp_result_list.append(f"{name}")

        tensors_vector_list_str = (
            "{ " + ",".join(amp_tensors_vector_list) + " }"
        )

        if len(amp_tensors_vector_list) == 0:
            layout_logic_str = ""
        else:
            after_call_str = f"{returns_type_str} {result_name} = {forward_function_name}({layout_inputs_call_args_str});\n"
            layout_logic_str = LAYOUT_LOGIC_TEMPLATE.format(
                tensors_vector_list_str,
                "    ".join(layout_tensors_vector_optional_list),
                "    ".join(layout_autotune_attr_code_list)
                + "    "
                + layout_autotune_list_str,
                after_call_str,
                "    ".join(layout_autotune_outs_list),
                returns_str,
            )

        return layout_logic_str

    def GenerateForwardDefinitionAndDeclaration(self, is_inplaced):
        namespace = self.namespace
        if self.forward_api_name[-1] == '_' and not is_inplaced:
            return
        forward_api_name = (
            GetInplacedFunctionName(self.forward_api_name)
            if is_inplaced
            else self.forward_api_name
        )

        forward_inputs_position_map = self.forward_inputs_position_map
        forward_outputs_position_map = self.forward_outputs_position_map
        forward_attrs_list = self.forward_attrs_list
        if not self.is_forward_only:
            backward_grad_outputs_map = self.backward_grad_outputs_map

        optional_inputs = self.optional_inputs
        intermediate_outputs = self.intermediate_outputs
        forward_inplace_map = self.forward_inplace_map if is_inplaced else {}
        indent = GetIndent(1)

        # Get Function Args
        num_inputs = len(forward_attrs_list) + len(
            forward_inputs_position_map.keys()
        )
        inputs_args_definition_list = ["" for i in range(num_inputs)]
        inputs_args_declaration_list = ["" for i in range(num_inputs)]
        inputs_call_list = ["" for i in range(num_inputs)]

        amp_inputs_call_list = ["" for i in range(num_inputs)]
        amp_tensors_vector_list = []
        amp_tensors_vector_optional_list = []
        amp_autocast_list = []
        amp_autocast_optional_list = []
        layout_autotune_list = []
        layout_autotune_optional_list = []
        layout_tensors_vector_optional_list = []
        for name, (ttype, pos) in forward_inputs_position_map.items():
            inputs_call_list[pos] = f"{name}"
            amp_inputs_call_list[pos] = f"new_{name}"
            is_optional = name in optional_inputs
            if IsPlainTensorType(ttype):
                if is_optional:
                    if (
                        self.is_forward_only
                        and is_inplaced
                        and forward_inplace_map
                        and name in forward_inplace_map.keys()
                    ):
                        arg_str = f"paddle::optional<paddle::experimental::Tensor>& {name}"
                    else:
                        arg_str = f"const paddle::optional<paddle::experimental::Tensor>& {name}"
                    amp_tensors_vector_optional_list.append(
                        f"if ({name}) amp_tensors_vector.push_back({{ *{name} }});\n"
                    )
                    amp_autocast_optional_list.append(
                        f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n"
                    )
                    layout_tensors_vector_optional_list.append(
                        f"if ({name}) tensors_vector.push_back({{ *{name} }});\n"
                    )
                    layout_autotune_optional_list.append(
                        f"auto new_{name} = transformer->TransInTensor(\"{name}\", {name});\n"
                    )
                else:
                    if (
                        is_inplaced
                        and forward_inplace_map
                        and name in forward_inplace_map.keys()
                    ):
                        arg_str = f"paddle::experimental::Tensor& {name}"
                        amp_tensors_vector_list.append(f"{{{name}}}")
                        amp_autocast_list.append(
                            f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n"
                        )
                    else:
                        arg_str = f"const paddle::experimental::Tensor& {name}"
                        amp_tensors_vector_list.append(f"{{{name}}}")
                        amp_autocast_list.append(
                            f"auto new_{name} = egr::EagerAmpAutoCast(\"{name}\", {name}, amp_dst_dtype, op_name);\n"
                        )
                    layout_autotune_list.append(
                        f"auto new_{name} = transformer->TransInTensor(\"{name}\", {name});\n"
                    )
            else:
                assert IsVectorTensorType(ttype)
                if is_optional:
                    if (
                        self.is_forward_only
                        and is_inplaced
                        and forward_inplace_map
                        and name in forward_inplace_map.keys()
                    ):
                        arg_str = f"paddle::optional<std::vector<paddle::experimental::Tensor>>& {name}"
                    else:
                        arg_str = f"const paddle::optional<std::vector<paddle::experimental::Tensor>>& {name}"
                    amp_tensors_vector_optional_list.append(
                        f"if ({name}) amp_tensors_vector.push_back( *{name} );\n"
                    )
                    amp_autocast_optional_list.append(
                        f"auto new_{name} = egr::EagerAmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n"
                    )
                    layout_autotune_optional_list.append(
                        f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n"
                    )
                else:
                    if (
                        is_inplaced
                        and forward_inplace_map
                        and name in forward_inplace_map.keys()
                    ):
                        arg_str = (
                            f"std::vector<paddle::experimental::Tensor>& {name}"
                        )
                    else:
                        arg_str = f"const std::vector<paddle::experimental::Tensor>& {name}"
                    amp_tensors_vector_list.append(f"{name}")
                    amp_autocast_list.append(
                        f"auto new_{name} = egr::EagerAmpAutoCasts(\"{name}\", {name}, amp_dst_dtype, op_name);\n"
                    )
                    layout_autotune_list.append(
                        f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n"
                    )

            inputs_args_definition_list[pos] = arg_str
            inputs_args_declaration_list[pos] = arg_str

        # forward attrs
        for name, atype, default_val, pos in forward_attrs_list:
            inputs_call_list[pos] = name
            amp_inputs_call_list[pos] = name
            if default_val is not None:
                inputs_args_declaration_list[
                    pos
                ] = f"{atype} {name} = {default_val}"
            else:
                inputs_args_declaration_list[pos] = f"{atype} {name}"
            inputs_args_definition_list[pos] = f"{atype} {name}"

        inputs_args_declaration_str = ", ".join(inputs_args_declaration_list)
        inputs_args_definition_str = ", ".join(inputs_args_definition_list)
        inputs_call_args_str = ", ".join(inputs_call_list)

        # Forward Full Logic
        function_name = forward_api_name
        if len(intermediate_outputs) > 0:
            if is_inplaced:
                function_name = (
                    GetIntermediateAPIFunctionName(forward_api_name[:-1]) + '_'
                )
            else:
                function_name = GetIntermediateAPIFunctionName(function_name)

        api_out_type = "auto"
        if is_inplaced and len(forward_outputs_position_map) == 1:
            api_out_type = "auto&"
        forward_call_str = f"{indent}{api_out_type} api_result = paddle::experimental::{namespace}{function_name}({inputs_call_args_str});"
        num_outputs = len(forward_outputs_position_map.keys()) - len(
            intermediate_outputs
        )

        # Check Nan and Inf
        check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format(
            function_name, "api_result"
        )

        # Get Outputs
        get_outputs_str = ""
        for name, (rtype, pos) in forward_outputs_position_map.items():
            if num_outputs == 1 and len(intermediate_outputs) == 0:
                get_outputs_str += f"{indent}auto& {name} = api_result;\n"
            else:
                get_outputs_str += (
                    f"{indent}auto& {name} = std::get<{pos}>(api_result);\n"
                )

        # Get return type list & outputs
        returns_type_list = ["" for i in range(num_outputs)]
        returns_list = ["" for i in range(num_outputs)]
        for name, (rtype, pos) in forward_outputs_position_map.items():
            if name in intermediate_outputs:
                continue
            returns_list[pos] = f"{name}"

            if IsPlainTensorType(rtype):
                if (
                    is_inplaced
                    and forward_inplace_map
                    and name in forward_inplace_map.values()
                ):
                    ind = list(forward_inplace_map.values()).index(name)
                    if (
                        list(forward_inplace_map.keys())[ind]
                        in self.optional_inputs
                    ):
                        returns_type_list[pos] = inplace_optional_out_type_map[
                            rtype
                        ]
                    else:
                        returns_type_list[pos] = "paddle::experimental::Tensor&"
                else:
                    returns_type_list[pos] = "paddle::experimental::Tensor"
            else:
                assert IsVectorTensorType(rtype)
                if (
                    is_inplaced
                    and forward_inplace_map
                    and name in forward_inplace_map.values()
                ):
                    ind = list(forward_inplace_map.values()).index(name)
                    if (
                        list(forward_inplace_map.keys())[ind]
                        in self.optional_inputs
                    ):
                        returns_type_list[pos] = inplace_optional_out_type_map[
                            rtype
                        ]
                    else:
                        returns_type_list[
                            pos
                        ] = "std::vector<paddle::experimental::Tensor>&"
                else:
                    returns_type_list[
                        pos
                    ] = "std::vector<paddle::experimental::Tensor>"

        if num_outputs == 1:
            returns_str = returns_list[0]
            returns_type_str = returns_type_list[0]
        else:
            returns_type_str = ", ".join(returns_type_list)
            returns_type_str = f"std::tuple<{returns_type_str}>"
            returns_str = ", ".join(returns_list)
            returns_str = f"{returns_type_str}{{{returns_str}}}"

        # Node Creation Pre-Processing
        inputs_names = []
        if not self.is_forward_only:
            # 1. Get Input AutoGradMeta
            inputs_autograd_meta_list = []
            compute_require_grad_args_list = ["trace_backward"]
            for name, (ttype, pos) in forward_inputs_position_map.items():
                # Has corresponding grad output
                has_corresponding_grad_output = False
                for _, (
                    _,
                    corresponding_pos,
                    _,
                ) in backward_grad_outputs_map.items():
                    if pos == corresponding_pos:
                        has_corresponding_grad_output = True
                if (
                    has_corresponding_grad_output
                    or (
                        name in forward_inplace_map
                        and forward_api_name not in inplace_check_blacklist
                    )
                    or self.is_forward_only
                ):
                    input_autograd_meta_name = GetAutoGradMetaName(name)
                    if IsPlainTensorType(ttype):
                        input_autograd_meta = f"{indent}egr::AutogradMeta* {input_autograd_meta_name} = egr::EagerUtils::nullable_autograd_meta({name});"
                    else:
                        assert IsVectorTensorType(ttype)
                        input_autograd_meta_vec_name = (
                            GetAutoGradMetaVectorName(name)
                        )
                        input_autograd_meta = f"{indent}std::vector<egr::AutogradMeta*> {input_autograd_meta_vec_name} = egr::EagerUtils::nullable_autograd_meta({name});\n"
                        input_autograd_meta += f"{indent}std::vector<egr::AutogradMeta*>* {input_autograd_meta_name} = &{input_autograd_meta_vec_name};"
                    inputs_autograd_meta_list.append(input_autograd_meta)
                    compute_require_grad_args_list.append(
                        input_autograd_meta_name
                    )

            inputs_autograd_meta_str = "\n".join(inputs_autograd_meta_list)
            compute_require_grad_args_str = ",".join(
                compute_require_grad_args_list
            )

            # 2. Get Output AutoGradMeta
            outputs_autograd_meta_list = []
            num_fwd_outputs = len(forward_outputs_position_map.keys())

            for name, (rtype, pos) in forward_outputs_position_map.items():
                output_autograd_meta_name = GetAutoGradMetaName(name)
                output_autograd_meta_vec_name = GetAutoGradMetaVectorName(name)
                if num_fwd_outputs == 1:
                    if IsPlainTensorType(rtype):
                        output_autograd_meta = f"{indent}egr::AutogradMeta* {output_autograd_meta_name} = egr::EagerUtils::autograd_meta(&{name});"
                    else:
                        assert IsVectorTensorType(rtype)
                        output_autograd_meta = f"{indent}std::vector<egr::AutogradMeta*> {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{name});\n"
                        output_autograd_meta += f"{indent}std::vector<egr::AutogradMeta*>* {output_autograd_meta_name} = &{output_autograd_meta_vec_name};"
                else:
                    # Tuple api_result
                    if IsPlainTensorType(rtype):
                        output_autograd_meta = f"{indent}egr::AutogradMeta* {output_autograd_meta_name} = egr::EagerUtils::autograd_meta(&{name});"
                    else:
                        assert IsVectorTensorType(rtype)
                        output_autograd_meta = f"{indent}std::vector<egr::AutogradMeta*> {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{name});\n"
                        output_autograd_meta += f"{indent}std::vector<egr::AutogradMeta*>* {output_autograd_meta_name} = &{output_autograd_meta_vec_name};"

                outputs_autograd_meta_list.append(output_autograd_meta)
            outputs_autograd_meta_str = "\n".join(outputs_autograd_meta_list)

            # 3. Check Inplace
            check_inplace_str = ""
            bump_inplace_version_str = ""
            if is_inplaced:
                for inplace_name in forward_inplace_map.keys():
                    if forward_api_name not in inplace_check_blacklist:
                        inplace_autograd_meta_name = GetAutoGradMetaName(
                            inplace_name
                        )
                        check_inplace_str += CHECK_INPLACE_TEMPLATE.format(
                            inplace_name, inplace_autograd_meta_name
                        )
                    bump_inplace_version_str += (
                        BUMP_INPLACE_VERSION_TEMPLATE.format(
                            inplace_name, inplace_name
                        )
                    )

            # Node Creation
            self.GenerateNodeCreationCodes()
            node_creation_str = self.node_creation_str

        dygraph_event_str = f"{indent}paddle::platform::RecordEvent dygraph_entrance_record_event(\"{forward_api_name} dygraph\", paddle::platform::TracerEventType::Operator, 1);\n"
        forward_ad_function_name = GetDygraphForwardFunctionName(
            forward_api_name
        )

        # Forward amp logic
        kernel_trans2_op_name_str = (
            f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");"
        )
        amp_tensors_vector_list_str = (
            "{ " + ",".join(amp_tensors_vector_list) + " }"
        )
        amp_tensors_vector_optional_list_str = "    ".join(
            amp_tensors_vector_optional_list
        )
        amp_get_dst_dtype_str = "auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector);\n"
        amp_autocast_list_str = (
            "    ".join(amp_autocast_list)
            + "    "
            + "    ".join(amp_autocast_optional_list)
        )
        amp_inputs_call_args_str = ", ".join(amp_inputs_call_list)
        amp_call_str = (
            f"return {forward_ad_function_name}({amp_inputs_call_args_str});"
        )
        if is_inplaced or (forward_api_name == "cast"):
            amp_logic_str = "\n VLOG(5) << \" No AMP for {} because it is a inplace or cast api. \"; ".format(
                forward_ad_function_name
            )
        else:
            amp_logic_str = AMP_LOGIC_TEMPLATE.format(
                kernel_trans2_op_name_str,
                amp_tensors_vector_list_str,
                amp_tensors_vector_optional_list_str,
                amp_get_dst_dtype_str,
                amp_autocast_list_str,
                amp_call_str,
            )

        # Forward layout autotune
        layout_autotune_list_str = "    ".join(
            layout_autotune_list
        ) + "    ".join(layout_autotune_optional_list)
        layout_logic_str = self.GenerateForwardLayoutAutotune(
            forward_api_name,
            amp_tensors_vector_list,
            layout_tensors_vector_optional_list,
            layout_autotune_list_str,
            returns_type_str,
            returns_str,
            amp_inputs_call_args_str,
        )

        # For inputs outputs prepare for logging
        var_str = f"\n{indent}  std::string input_str = \"\";"
        var_str += f"\n{indent}  std::string output_str = \"\";"
        for name, (ttype, pos) in forward_inputs_position_map.items():
            var_str += f"\n{indent}  const char* TENSOR_{name.upper()}_TEMPLATE = \" \\n( {name} , [%s]), \";"
            var_str += f"\n{indent}  std::string input_{name}_str = paddle::string::Sprintf(TENSOR_{name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({name}));"
            var_str += f"\n{indent}  input_str += input_{name}_str; "

        before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str)
        for name, (ttype, pos) in forward_outputs_position_map.items():
            var_str += f"\n{indent}  const char* TENSOR_{name.upper()}_TEMPLATE = \" \\n( {name} , [%s]), \";"
            var_str += f"\n{indent}  std::string output_{name}_str = paddle::string::Sprintf(TENSOR_{name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({name}));"
            var_str += f"\n{indent}  output_str += output_{name}_str; "

        log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str)

        # Generate forward_definition_str and forward_declaration_str
        if self.is_forward_only:
            if len(amp_tensors_vector_list) == 0:
                amp_logic_str = "\n VLOG(7) << \" No AMP for {} because it has no input. \"; ".format(
                    forward_ad_function_name
                )
            self.forward_definition_str += (
                FORWARD_ONLY_FUNCTION_TEMPLATE.format(
                    returns_type_str,
                    forward_ad_function_name,
                    inputs_args_definition_str,
                    forward_api_name,
                    dygraph_event_str,
                    amp_logic_str,
                    layout_logic_str,
                    forward_api_name,
                    before_log_str,
                    forward_call_str,
                    get_outputs_str,
                    forward_api_name,
                    log_str,
                    returns_str,
                )
            )
        else:
            self.forward_definition_str += FORWARD_FUNCTION_TEMPLATE.format(
                returns_type_str,
                forward_ad_function_name,
                inputs_args_definition_str,
                forward_api_name,
                dygraph_event_str,
                amp_logic_str,
                layout_logic_str,
                inputs_autograd_meta_str,
                forward_api_name,
                before_log_str,
                forward_call_str,
                check_nan_inf_str,
                get_outputs_str,
                outputs_autograd_meta_str,
                compute_require_grad_args_str,
                check_inplace_str,
                bump_inplace_version_str,
                node_creation_str,
                forward_api_name,
                log_str,
                returns_str,
            )

        self.forward_declaration_str += f"{returns_type_str} {forward_ad_function_name}({inputs_args_declaration_str});\n"

    def GenerateInplacedForwardDygraphFunctions(self):
        # Inplaced Version Dygraph Function Generation
        forward_api_name = self.forward_api_name
        forward_api_contents = self.forward_api_contents

        if (
            forward_api_name != "sum"
            and "inplace" in forward_api_contents.keys()
        ):
            # Function Definition and Declaration Generation
            self.GenerateForwardDefinitionAndDeclaration(is_inplaced=True)
            self.UpdateCoreOpsInformation(is_inplaced=True)

    def UpdateCoreOpsInformation(self, is_inplaced):
        forward_api_name = (
            GetInplacedFunctionName(self.forward_api_name)
            if is_inplaced
            else self.forward_api_name
        )
        forward_inputs_position_map = self.forward_inputs_position_map
        forward_outputs_position_map = self.forward_outputs_position_map
        forward_attrs_list = self.forward_attrs_list

        num_args = len(forward_inputs_position_map.keys()) + len(
            forward_attrs_list
        )
        num_returns = len(forward_outputs_position_map.keys())

        fwd_api_name = "" + forward_api_name
        core_ops_returns_info[fwd_api_name] = ["" for i in range(num_returns)]
        core_ops_args_info[fwd_api_name] = ["" for i in range(num_args)]
        core_ops_args_type_info[fwd_api_name] = ["" for i in range(num_args)]

        for name, (ttype, pos) in forward_inputs_position_map.items():
            core_ops_args_info[fwd_api_name][pos] = name
            if IsPlainTensorType(ttype):
                core_ops_args_type_info[fwd_api_name][pos] = "tensor"
            else:
                assert IsVectorTensorType(ttype)
                core_ops_args_type_info[fwd_api_name][pos] = "list"

        for name, _, _, pos in forward_attrs_list:
            core_ops_args_info[fwd_api_name][pos] = name

        for name, (ttype, pos) in forward_outputs_position_map.items():
            core_ops_returns_info[fwd_api_name][pos] = name

    def run(self):
        super().run()

        #####################
        ## Code Generation ##
        #####################

        # Definition And Declaration
        self.GenerateForwardDefinitionAndDeclaration(is_inplaced=False)

        self.UpdateCoreOpsInformation(is_inplaced=False)

        self.GenerateInplacedForwardDygraphFunctions()


class DygraphNodeGenerator(DygraphFunctionGeneratorBase):
    def __init__(
        self,
        forward_api_contents,
        grad_api_contents,
        forward_apis_dict,
        namespace,
        next_grad_api_contents=None,
    ):
        DygraphFunctionGeneratorBase.__init__(
            self,
            forward_api_contents,
            grad_api_contents,
            forward_apis_dict,
            namespace,
        )

        # Record name mapping from forward_var_name to grad_var_names
        self.to_next_grad_name_mapping = {}  # {name : name}

        # Generated Results
        self.node_declaration_str = ""
        self.node_definition_str = ""
        self.next_grad_api_contents = next_grad_api_contents

    def TransformToNextGradName(self, string):
        name_mapping = self.to_next_grad_name_mapping
        if string in name_mapping.keys():
            return name_mapping[string]
        return string

    def ResetOptionalInputs(self):
        namespace = self.namespace
        grad_api_contents = self.grad_api_contents

        base_generator = FunctionGeneratorBase(grad_api_contents, namespace)
        base_generator.ParseDispensable()

        self.optional_inputs = base_generator.optional_inputs

    def RecordGrad2NextGradNameMapping(self, next_node_generator):
        next_orig_inputs_list = next_node_generator.orig_forward_inputs_list
        next_orig_returns_list = next_node_generator.orig_forward_returns_list

        next_forward_inputs_list = next_node_generator.forward_inputs_list
        next_forward_returns_list = next_node_generator.forward_returns_list
        for i in range(len(next_orig_inputs_list)):
            grad_name = next_orig_inputs_list[i][0]
            next_forward_name = next_forward_inputs_list[i][0]
            self.to_next_grad_name_mapping[grad_name] = next_forward_name

        for i in range(len(next_orig_returns_list)):
            grad_ret_name = next_orig_returns_list[i][0]
            next_ret_name = next_forward_returns_list[i][0]
            self.to_next_grad_name_mapping[grad_ret_name] = next_ret_name

    def GenerateHigherOrderNodeCreationCode(self):
        has_higher_order_node = False
        namespace = self.namespace
        grad_api_contents = self.grad_api_contents
        forward_apis_dict = self.forward_apis_dict
        next_grad_api_contents = self.next_grad_api_contents

        next_grad_node_creation_str = ""
        next_grad_node_out_list = []
        next_node_generator = None
        if next_grad_api_contents:
            # Fake forward_api_contents and backward_api_contents
            forward_api_contents = grad_api_contents
            forward_api_contents['op'] = forward_api_contents['backward_op']
            backward_api_contents = next_grad_api_contents

            next_node_generator = DygraphFunctionGeneratorBase(
                forward_api_contents,
                backward_api_contents,
                forward_apis_dict,
                namespace,
            )
            next_node_generator.run()
            next_node_generator.GenerateNodeCreationCodes(True)

            next_grad_node_creation_str = next_node_generator.node_creation_str
            next_grad_node_out_list = next_node_generator.grad_node_out_list

            self.RecordGrad2NextGradNameMapping(next_node_generator)

        is_invoke_forward_api = IsInvokeForwardApi(
            self.grad_api_contents, self.forward_apis_dict
        )
        if next_node_generator is not None:
            has_higher_order_node = True
            return (
                has_higher_order_node,
                is_invoke_forward_api,
                next_grad_node_creation_str,
                next_grad_node_out_list,
                next_node_generator.backward_forward_inputs_map,
            )
        elif not is_invoke_forward_api:
            next_grad_node_creation_str = f"""  if(trace_backward) {{
    PADDLE_THROW(phi::errors::Unavailable(
    \"The Op {self.backward_api_name} doesn't have any grad\"
    \"op. If you don't intend calculating higher order\"
    \"derivatives, please set `create_graph`to False.\"));
  }}"""
        return (
            has_higher_order_node,
            is_invoke_forward_api,
            next_grad_node_creation_str,
            next_grad_node_out_list,
            None,
        )

    def GenerateNodeDeclaration(self):
        forward_op_name = self.forward_api_name
        backward_forward_inputs_map = self.backward_forward_inputs_map
        backward_attrs_list = self.backward_attrs_list
        no_need_buffers = self.no_need_buffers

        # SetTensorWrapper Methods & TensorWrapper Members & ClearTensorWrappers
        set_tensor_wrapper_methods_str = ""
        tensor_wrapper_members_str = ""
        clear_tensor_wrapper_str = ""
        for tname, (
            ttype,
            is_fwd_input,
            _,
        ) in backward_forward_inputs_map.items():
            no_need_buffer = "true" if tname in no_need_buffers else "false"
            tensor_wrapper_name = GetSavedName(tname)
            if IsPlainTensorType(ttype):
                set_tensor_wrapper_methods_str += (
                    SET_PLAIN_TENSOR_WRAPPER_TEMPLATE.format(
                        tname, tname, tensor_wrapper_name, tname, no_need_buffer
                    )
                )

                tensor_wrapper_members_str += (
                    PLAIN_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name)
                )

                clear_tensor_wrapper_str += (
                    CLEAR_TENSOR_WRAPPER_TEMPLATE.format(tensor_wrapper_name)
                )

            else:
                assert IsVectorTensorType(ttype)
                set_tensor_wrapper_methods_str += (
                    SET_VECTOR_TENSOR_WRAPPER_TEMPLATE.format(
                        tname, tname, tname, tensor_wrapper_name, no_need_buffer
                    )
                )

                tensor_wrapper_members_str += (
                    VECTOR_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name)
                )

                clear_tensor_wrapper_str += (
                    CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE.format(
                        tensor_wrapper_name
                    )
                )

        # SetAttributes & Attribute Members
        set_attribute_methods_str = ""
        attribute_members_str = ""
        for aname, atype, default_val, _ in backward_attrs_list:
            saved_attr_name = GetSavedName(aname)
            set_attribute_methods_str += SET_ATTR_METHOD_TEMPLATE.format(
                aname, GetConstReference(atype), aname, saved_attr_name, aname
            )

            if default_val:
                attribute_members_str += (
                    ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE.format(
                        RemoveConstAndReference(atype),
                        saved_attr_name,
                        default_val,
                    )
                )
            else:
                attribute_members_str += ATTRIBUTE_MEMBER_TEMPLATE.format(
                    RemoveConstAndReference(atype), saved_attr_name
                )

        grad_node_name = GetGradNodeName(self.backward_api_name)
        self.node_declaration_str = NODE_DECLARATION_TEMPLATE.format(
            grad_node_name,
            grad_node_name,
            grad_node_name,
            grad_node_name,
            grad_node_name,
            clear_tensor_wrapper_str,
            grad_node_name,
            grad_node_name,
            set_tensor_wrapper_methods_str,
            set_attribute_methods_str,
            tensor_wrapper_members_str,
            attribute_members_str,
        )

    def GenerateNodeDefinition(
        self,
        has_higher_order_node,
        is_invoke_forward_api,
        next_grad_node_creation_str,
        next_grad_node_out_list,
        backward_forward_inputs_map_next,
    ):
        namespace = self.namespace
        forward_api_name = self.forward_api_name
        backward_api_name = self.backward_api_name
        backward_forward_inputs_map = self.backward_forward_inputs_map
        backward_grad_inputs_map = self.backward_grad_inputs_map
        backward_grad_outputs_map = self.backward_grad_outputs_map
        backward_attrs_list = self.backward_attrs_list
        backward_inplace_map = self.backward_inplace_map
        indent = GetIndent(1)

        # Construct grad_api function args
        # Order: TensorWrappers, GradTensors, Attributes
        grad_api_args_len = (
            len(backward_forward_inputs_map.keys())
            + len(backward_grad_inputs_map.keys())
            + len(backward_attrs_list)
        )
        grad_api_args = ["" for i in range(grad_api_args_len)]
        get_grad_in_args_list = []

        # Fill Grad Ins with Zero
        fill_zero_str = ""
        if backward_api_name in ops_to_fill_zero_for_empty_grads:
            fill_zero_str = (
                f"{indent}const auto& input_metas = this->InputMeta();\n"
            )
            for name, (
                ttype,
                fwd_position,
                grad_api_position,
            ) in backward_grad_inputs_map.items():
                if name in self.optional_inputs:
                    if IsPlainTensorType(ttype):
                        fill_zero_str += f"{indent}egr::EagerUtils::FillZeroForEmptyOptionalGradInput(&grads[{fwd_position}][0], input_metas[{fwd_position}][0]);\n"
                else:
                    if IsPlainTensorType(ttype):
                        fill_zero_str += f"{indent}egr::EagerUtils::FillZeroForEmptyGradInput(&grads[{fwd_position}][0], input_metas[{fwd_position}][0]);\n"
                    else:
                        fill_zero_str += f"{indent}egr::EagerUtils::FillZeroForEmptyGradInput(&grads[{fwd_position}], input_metas[{fwd_position}]);\n"

        inplace_grad_input_str = ""
        inplaced_tensor_wrapper = False
        inplace_check_str = ""
        optional_inplace_var_name = []
        # Grad Ins from TensorWrappers
        for (
            name,
            (backward_input_type, is_fwd_input, grad_api_position),
        ) in backward_forward_inputs_map.items():
            tensor_wrapper_name = GetSavedName(name)
            transformed_tensor_name = self.TransformToNextGradName(name)

            is_optional = name in self.optional_inputs
            tensor_wrapper_recover_str = f"{indent}auto {transformed_tensor_name} = egr::EagerUtils::RecoverTensorWrapper(&this->{tensor_wrapper_name});"
            if backward_inplace_map and name in backward_inplace_map.keys():
                if has_higher_order_node:
                    if (
                        transformed_tensor_name
                        in backward_forward_inputs_map_next
                    ) and (
                        backward_forward_inputs_map_next[
                            transformed_tensor_name
                        ][1]
                    ):
                        optional_inplace_var_name.append(
                            transformed_tensor_name
                        )
                tensor_wrapper_intermidiate_tensor_str = (
                    f"(&this->{tensor_wrapper_name})->get_intermidiate_tensor()"
                )
                inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format(
                    transformed_tensor_name,
                    transformed_tensor_name,
                    name,
                    transformed_tensor_name,
                    transformed_tensor_name,
                    transformed_tensor_name,
                    transformed_tensor_name,
                    tensor_wrapper_intermidiate_tensor_str,
                )
                inplace_grad_input_str = transformed_tensor_name
            if is_optional:
                if backward_input_type == "std::vector<Tensor>":
                    tensor_wrapper_recover_str += (
                        "\n"
                        + CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE.format(
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                        )
                    )
                else:
                    tensor_wrapper_recover_str += (
                        "\n"
                        + CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE.format(
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                        )
                    )

                grad_api_args[grad_api_position] = (
                    transformed_tensor_name + "_optional"
                )

            else:
                grad_api_args[grad_api_position] = transformed_tensor_name

            get_grad_in_args_list.append(tensor_wrapper_recover_str)

        # Grad Ins from grads
        for name, (
            ttype,
            fwd_position,
            grad_api_position,
        ) in backward_grad_inputs_map.items():
            transformed_tensor_name = self.TransformToNextGradName(name)

            is_optional = name in self.optional_inputs
            if IsPlainTensorType(ttype):
                get_tensor_str = f"{indent}auto& {transformed_tensor_name} = hooked_grads[{fwd_position}][0];"

                # Inplace in backward op
                if backward_inplace_map and name in backward_inplace_map.keys():
                    if has_higher_order_node:
                        if (
                            transformed_tensor_name
                            in backward_forward_inputs_map_next
                        ) and (
                            backward_forward_inputs_map_next[
                                transformed_tensor_name
                            ][1]
                        ):
                            optional_inplace_var_name.append(
                                transformed_tensor_name
                            )
                    grads_tensor_str = f"grads[{fwd_position}][0]"
                    inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format(
                        transformed_tensor_name,
                        transformed_tensor_name,
                        name,
                        transformed_tensor_name,
                        transformed_tensor_name,
                        transformed_tensor_name,
                        transformed_tensor_name,
                        grads_tensor_str,
                    )
                    inplace_grad_input_str = transformed_tensor_name

                if is_optional:
                    get_tensor_str += (
                        "\n"
                        + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format(
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                            transformed_tensor_name,
                        )
                    )
                    grad_api_args[
                        grad_api_position
                    ] = f"{transformed_tensor_name}_optional"
                else:
                    grad_api_args[grad_api_position] = transformed_tensor_name
            else:
                assert IsVectorTensorType(ttype)
                get_tensor_str = f"{indent}auto& {transformed_tensor_name} = hooked_grads[{fwd_position}];"
                grad_api_args[grad_api_position] = transformed_tensor_name

            get_grad_in_args_list.append(get_tensor_str)

        # Grad Attrs
        for name, _, _, grad_api_position in backward_attrs_list:
            saved_attribute_name = GetSavedName(name)
            get_attr_str = (
                f"{indent}auto& {name} = this->{saved_attribute_name};"
            )

            grad_api_args[grad_api_position] = name
            get_grad_in_args_list.append(get_attr_str)

        get_grad_in_args_str = "\n".join(get_grad_in_args_list)

        # Grad Function Call String
        slot_num_bwd_outputs = len(self.forward_inputs_position_map.keys())
        grad_api_namespace = f"paddle::experimental::{namespace}"
        grad_function_prepare_str = f"""
  const auto& out_metas = OutputMeta();
  paddle::small_vector<std::vector<paddle::experimental::Tensor>, egr::kSlotSmallVectorSize> returns({slot_num_bwd_outputs});
  for (int i = 0; i < {slot_num_bwd_outputs}; ++i) {{
    out_metas[i].size() == 0 ? returns[i].resize(1) : returns[i].resize(out_metas[i].size());
  }}
"""
        inplace_for_grad_outs_str = ""
        optional_inplace_str = ""
        # Grad Outputs
        out_index = -1
        out_assign_str = ""
        for name, (
            ttype,
            fwd_position,
            grad_api_position,
        ) in backward_grad_outputs_map.items():
            transformed_tensor_name = self.TransformToNextGradName(name)
            out_index = out_index + 1
            if is_invoke_forward_api:
                if len(backward_grad_outputs_map) == 1:
                    out_assign_str += (
                        f"{indent}*api_output_{out_index} = api_output;\n"
                    )
                else:
                    out_assign_str += f"{indent}*api_output_{out_index} = std::get<{out_index}>(api_output);\n"
            else:
                grad_api_args.append(f"api_output_{out_index}")
            if inplace_grad_input_str in optional_inplace_var_name:
                optional_inplace_str = "VLOG(6) << \"No Inplace should happend for wrappered input: {inplace_grad_input_str}\";"
            else:
                optional_inplace_str = f"""if (api_output_{out_index} != nullptr && can_be_inplaced) {{
      egr::EagerUtils::HandleViewBetweenInputAndOutput({inplace_grad_input_str}, api_output_{out_index});
    }}"""
            if IsPlainTensorType(ttype):

                if (
                    backward_inplace_map
                    and name in backward_inplace_map.values()
                ):
                    inplace_str = f""" if (api_output_{out_index} != nullptr && can_be_inplaced) {{
      egr::EagerUtils::HandleViewBetweenInputAndOutput({inplace_grad_input_str}, api_output_{out_index});
    }}"""
                    if has_higher_order_node:
                        inplace_for_grad_outs_str += f"""
  if (trace_backward) {{
    {optional_inplace_str}
  }} else {{
    {inplace_str}
  }}"""
                    else:
                        inplace_for_grad_outs_str += inplace_str

                grad_function_prepare_str += f"""
  auto* api_output_{out_index} = (out_metas[{fwd_position}].empty() || out_metas[{fwd_position}][0].IsStopGradient()) ? nullptr : &returns[{fwd_position}][0];"""

            else:
                assert IsVectorTensorType(ttype)
                grad_function_prepare_str += f"""
  std::vector<paddle::experimental::Tensor*> api_output_{out_index};
  api_output_{out_index}.reserve(returns[{fwd_position}].size());
  for (size_t i = 0; i < returns[{fwd_position}].size(); ++i) {{
    if (out_metas[{fwd_position}].empty() || out_metas[{fwd_position}][i].IsStopGradient()) {{
      api_output_{out_index}.push_back(nullptr);
    }} else {{
      api_output_{out_index}.push_back(&returns[{fwd_position}][i]);
    }}
  }}"""

        grad_api_args_str = ", ".join(grad_api_args)

        if is_invoke_forward_api:
            autograd_api_out = "auto"
            if (
                len(self.backward_inplace_map) > 0
                and len(backward_grad_outputs_map) == 1
            ):
                autograd_api_out = "auto&"
            forward_api_name = (
                self.grad_api_contents['invoke'].split('(')[0].strip()
            )
            autograd_api = self.grad_api_contents['invoke'].replace(
                forward_api_name,
                GetDygraphForwardFunctionName(forward_api_name),
                1,
            )
            grad_function_call_str = f"""
  if (trace_backward) {{
  {indent}{autograd_api_out} api_output = {autograd_api};
  {out_assign_str}}} else {{
  {indent}{autograd_api_out} api_output = paddle::experimental::{self.namespace}{self.grad_api_contents['invoke']};
  {out_assign_str}{indent}}}
  """
        else:
            grad_function_call_str = f"""
{indent}{grad_api_namespace}{backward_api_name}({grad_api_args_str});"""

        # Check Nan and Inf
        check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format(
            backward_api_name, "returns"
        )

        # Prepare for Node Creation if Necessary
        outputs_autograd_meta_str = ""
        compute_require_next_grad_str = ""
        if len(next_grad_node_creation_str) > 0 or is_invoke_forward_api:
            compute_require_next_grad_str = f"{indent}bool trace_backward = egr::Controller::Instance().HasGrad() && create_graph;\n"

        # 3. Get Output AutoGradMeta
        outputs_autograd_meta_list = []
        # TODO(jiabin): Optimize this with SetStopGradient instead of Pass Stop gradient

        num_fwd_outputs = len(backward_grad_outputs_map.keys())
        for name, (
            rtype,
            pos,
            grad_api_position,
        ) in backward_grad_outputs_map.items():
            transformed_tensor_name = self.TransformToNextGradName(name)

            output_autograd_meta_name = GetAutoGradMetaName(
                transformed_tensor_name
            )
            output_autograd_meta_vec_name = GetAutoGradMetaVectorName(
                transformed_tensor_name
            )
            if IsPlainTensorType(rtype):
                output_autograd_meta = f"""
  auto& {transformed_tensor_name} = returns[{pos}][0];
  egr::AutogradMeta* {output_autograd_meta_name} = returns[{pos}][0].initialized() ? egr::EagerUtils::autograd_meta(&{transformed_tensor_name}) : nullptr;
  if ({output_autograd_meta_name}) {output_autograd_meta_name}->SetStopGradient(false);
  """

            else:
                assert IsVectorTensorType(rtype)
                if has_higher_order_node > 0:
                    output_autograd_meta = f"""
    auto& {transformed_tensor_name} = returns[{pos}];
    std::vector<egr::AutogradMeta*> {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{transformed_tensor_name});
    std::vector<egr::AutogradMeta*>* {output_autograd_meta_name} = &{output_autograd_meta_vec_name};
    for(auto* meta : {output_autograd_meta_vec_name}){{
        meta->SetStopGradient(false);
    }}
"""
                else:
                    output_autograd_meta = f"""
    auto& {transformed_tensor_name} = returns[{pos}];
    std::vector<egr::AutogradMeta*> {output_autograd_meta_vec_name} = egr::EagerUtils::autograd_meta(&{transformed_tensor_name});
    for(auto* meta : {output_autograd_meta_vec_name}){{
        meta->SetStopGradient(false);
    }}
"""
            outputs_autograd_meta_list.append(output_autograd_meta)

        outputs_autograd_meta_str = "\n".join(outputs_autograd_meta_list)

        returns_str = f"{indent}if(NeedComplexToRealConversion()) HandleComplexGradToRealGrad(&returns);\n"
        returns_str += f"{indent}return returns;\n"

        grad_node_name = GetGradNodeName(self.backward_api_name)
        # For inputs outputs prepare for logging
        var_str = f"\n{indent}  std::string input_str = \"\";"
        var_str += f"\n{indent}  std::string output_str = \"\";"
        for name, (
            ttype,
            fwd_position,
            grad_api_position,
        ) in backward_grad_inputs_map.items():
            new_name = self.TransformToNextGradName(name)
            var_str += f"\n{indent}  const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";"
            var_str += f"\n{indent}  std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));"
            var_str += f"\n{indent}  input_str += input_{new_name}_str; "

        for (
            name,
            (backward_input_type, is_fwd_input, grad_api_position),
        ) in backward_forward_inputs_map.items():
            new_name = self.TransformToNextGradName(name)
            var_str += f"\n{indent}  const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";"
            var_str += f"\n{indent}  std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));"
            var_str += f"\n{indent}  input_str += input_{new_name}_str; "

        before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str)

        for name, (
            ttype,
            fwd_position,
            grad_api_position,
        ) in backward_grad_outputs_map.items():
            new_name = self.TransformToNextGradName(name)
            var_str += f"\n{indent}  const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n ( {new_name} , [%s]), \";"
            var_str += f"\n{indent}  std::string output_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));"
            var_str += f"\n{indent}  output_str += output_{new_name}_str; "

        log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str)

        self.node_definition_str = GRAD_FUNCTION_TEMPLATE.format(
            grad_node_name,
            self.backward_api_name,
            fill_zero_str,
            get_grad_in_args_str,
            grad_function_prepare_str,
            compute_require_next_grad_str,
            inplace_check_str,
            inplace_for_grad_outs_str,
            self.backward_api_name,
            before_log_str,
            grad_function_call_str,
            check_nan_inf_str,
            outputs_autograd_meta_str,
            next_grad_node_creation_str,
            self.backward_api_name,
            log_str,
            returns_str,
        )

    def run(self):
        super().run()

        self.ResetOptionalInputs()

        #####################
        ## Code Generation ##
        #####################
        # Higher-order GradNode generation
        (
            has_higher_order_node,
            is_invoke_forward_api,
            next_grad_node_creation_str,
            next_grad_node_out_list,
            backward_forward_inputs_map,
        ) = self.GenerateHigherOrderNodeCreationCode()

        self.GenerateNodeDeclaration()

        self.GenerateNodeDefinition(
            has_higher_order_node,
            is_invoke_forward_api,
            next_grad_node_creation_str,
            next_grad_node_out_list,
            backward_forward_inputs_map,
        )


class DygraphForwardAndNodesGenerator(GeneratorBase):
    def __init__(self, api_yaml_path, backward_yaml_path):
        # Parent members:
        # self.namespace
        # self.api_yaml_path
        # self.forward_api_list
        GeneratorBase.__init__(self, api_yaml_path)

        self.backward_yaml_path = backward_yaml_path
        self.grad_api_dict = {}

        self.forward_declaration_str = ""
        self.forward_definition_str = ""

        self.node_declaration_str = ""
        self.node_definition_str = ""

    def CollectIsForwardOnly(self, forward_api_contents):
        self.is_forward_only = (
            False if 'backward' in forward_api_contents.keys() else True
        )

    def ParseYamlContents(self):
        self.ParseForwardYamlContents()

        backward_yaml_path = self.backward_yaml_path

        # string api is forward_only, no backward_yaml respectively
        if backward_yaml_path is not None:
            self.grad_api_dict = ReadBwdFile(backward_yaml_path)

    def GetBackwardAPIContents(self, forward_api_contents):
        grad_api_dict = self.grad_api_dict

        if 'backward' not in forward_api_contents.keys():
            return None

        backward_api_name = forward_api_contents['backward']
        assert backward_api_name in grad_api_dict.keys(), AssertMessage(
            backward_api_name, grad_api_dict.keys()
        )
        backward_api_contents = grad_api_dict[backward_api_name]

        return backward_api_contents

    def GenerateCode(self):
        forward_api_list = self.forward_api_list
        grad_api_dict = self.grad_api_dict
        forward_apis_dict = {}
        for api_item in forward_api_list:
            forward_apis_dict[api_item['op']] = api_item
        namespace = self.namespace

        for forward_api_contents in forward_api_list:
            if forward_api_contents['op'] in black_ops_list:
                continue

            self.CollectIsForwardOnly(forward_api_contents)

            if self.is_forward_only:
                backward_api_contents = None
            else:
                backward_api_contents = self.GetBackwardAPIContents(
                    forward_api_contents
                )

            # Generate Dygraph Forward Function
            function_generator = DygraphForwardFunctionGenerator(
                forward_api_contents,
                backward_api_contents,
                forward_apis_dict,
                namespace,
            )
            function_generator.run()

            self.forward_definition_str += (
                function_generator.forward_definition_str + "\n"
            )
            self.forward_declaration_str += (
                function_generator.forward_declaration_str + "\n"
            )

            # Generate Dygraph GradNode Function
            while True:
                if backward_api_contents is None:
                    break
                next_grad_api_contents = self.GetBackwardAPIContents(
                    backward_api_contents
                )

                node_generator = DygraphNodeGenerator(
                    forward_api_contents,
                    backward_api_contents,
                    forward_apis_dict,
                    namespace,
                    next_grad_api_contents,
                )
                node_generator.run()
                self.node_declaration_str += (
                    node_generator.node_declaration_str + "\n"
                )
                self.node_definition_str += (
                    node_generator.node_definition_str + "\n"
                )

                if next_grad_api_contents is None:
                    break

                # Detect if there exists higher-order GradNode
                forward_api_contents = backward_api_contents

                # Fake forward_api_content
                forward_api_contents['op'] = forward_api_contents['backward_op']
                backward_api_contents = next_grad_api_contents

        if len(namespace) > 0:
            if namespace.endswith("::"):
                namespace = namespace[:-2]
            self.forward_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format(
                namespace, self.forward_definition_str
            )
            self.forward_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format(
                namespace, self.forward_declaration_str
            )
            self.node_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format(
                namespace, self.node_declaration_str
            )
            self.node_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format(
                namespace, self.node_definition_str
            )

    def run(self):
        self.ParseYamlContents()

        self.InferNameSpace()

        self.GenerateCode()


##################
## File Writers ##
##################
def GenerateNodeCCFile(filepath, node_definition_str):
    if os.path.exists(filepath):
        os.remove(filepath)

    file_contents = NODE_CC_FILE_TEMPLATE.format(node_definition_str)
    with open(filepath, 'a') as f:
        f.write(file_contents)


def GenerateNodeHFile(filepath, node_declaration_str):
    if os.path.exists(filepath):
        os.remove(filepath)

    file_contents = NODE_H_FILE_TEMPLATE.format(node_declaration_str)
    with open(filepath, 'a') as f:
        f.write(file_contents)


def GenerateForwardCCFile(filepath, forward_definition_str):
    if os.path.exists(filepath):
        os.remove(filepath)

    core_ops_info_str = GenerateCoreOpInfoDefinition()
    file_contents = FORWARD_CC_FILE_TEMPLATE.format(
        core_ops_info_str, forward_definition_str
    )
    with open(filepath, 'a') as f:
        f.write(file_contents)


def GenerateForwardHFile(filepath, forward_function_declaration_str):
    if os.path.exists(filepath):
        os.remove(filepath)

    core_ops_info_str = GenerateCoreOpInfoDeclaration()
    file_contents = FORWARD_H_FILE_TEMPLATE.format(
        core_ops_info_str, forward_function_declaration_str
    )
    with open(filepath, 'a') as f:
        f.write(file_contents)


if __name__ == "__main__":
    args = ParseArguments()

    api_yaml_paths = args.api_yaml_path.split(",")
    backward_yaml_paths = args.backward_yaml_path.split(",")

    # Generate per Dygraph API
    node_declaration_str = ""
    node_definition_str = ""

    forward_declaration_str = ""
    forward_definition_str = ""

    for i in range(len(api_yaml_paths)):
        api_yaml_path = api_yaml_paths[i]

        # string api is forwrad only
        if not api_yaml_path.endswith('strings_ops.yaml'):
            backward_yaml_path = backward_yaml_paths[i]
        else:
            backward_yaml_path = None

        generator = DygraphForwardAndNodesGenerator(
            api_yaml_path, backward_yaml_path
        )
        generator.run()

        node_declaration_str += generator.node_declaration_str + "\n"
        node_definition_str += generator.node_definition_str + "\n"

        forward_declaration_str += generator.forward_declaration_str + "\n"
        forward_definition_str += generator.forward_definition_str + "\n"

    # Generate Files
    nodes_h_path = args.nodes_h_path
    nodes_cc_path = args.nodes_cc_path
    forwards_h_path = args.forwards_h_path
    forwards_cc_path = args.forwards_cc_path

    GenerateNodeCCFile(nodes_cc_path, node_definition_str)
    GenerateNodeHFile(nodes_h_path, node_declaration_str)
    GenerateForwardCCFile(forwards_cc_path, forward_definition_str)
    GenerateForwardHFile(forwards_h_path, forward_declaration_str)