[cherry-pick][Dy2stat] training with @declarative decorator and save_inference_model (#24557)

* [Dy2Stat] Add test for ptb model. (#24076)

* [Dy2Stat] Add test for ptb model. test=develop

* Simplify code for gast.If in is_control_flow_to_transform. test=develop

* Move IsControlFlowVisitor to file utils. test=develop

* Don't use convert_call for build-in func in CallTransformer. test=develop

* Optimize api is_control_flow_to_transform. test=develop

* Polish the document of IsControlFlowVisitor. test=develop

* Use declarative instead of dygraph_to_static_func. test=develop

* [dy2static] Add print transformer and unify print format (#24068)

* add print transformer & unify print format, test=develop

* remove using of dygraph_to_static_func, test=develop

* remove python stdout capture, test=develop

* fix compatibility problems for PY2, test=develop

* fix detail error, test=develop

* fix type analysis bug, test=develop

* fix print tuple compatible error in PY2, test=develop

* replace get_func to declarative, test=develop

* fix detail bug, test=develop

* fix some detail problems, test=develop

* change visit_call in print transformer, test=develop

* [dy2static] Support for static graph training with @declarative decorator (#24259)

* support to train in static

* support to independent decorator

* remove in_dygraph_mode condition in ProgramTranslator

* fix import param_guard and add train/eval test=develop

* Modify into ShareVarsFromScope and rm __all__ in partial_program test=develop

* [Dy2Stat] Optimize loop cond (#24049)

* Simplify code for gast.If in is_control_flow_to_transform.
* Move IsControlFlowVisitor to file utils. 
* Don't use convert_call for build-in func in CallTransformer. 
* Optimize api is_control_flow_to_transform. 
* Polish the document of IsControlFlowVisitor.

* revert modification from #24259

* [dy2stat]Support save_inference_model in program_translator (#24353)

* support save_inference_model in program_translator test=develop

* fix compatibility with OrderedDict.values() in python3 test=develop

* synchronized random_seed test=develop

* Polish Error Message test=develop

* Fix bug with `if Tensor` in is_control_flow (#24433)

* fix bug with `if Tensor` in is_control_flow test=develop

* remove continue test=develop

* Revert "[dy2static] Add print transformer and unify print format (#24068)"

This reverts commit 09dd0190.

* Revert "[dy2static] Add print transformer and unify print format (#24068)"

This reverts commit 09dd0190.

* fix sample code in sava_inference_model test=develop
Co-authored-by: Nliym27 <33742067+liym27@users.noreply.github.com>
Co-authored-by: NChen Weihang <chenweihang@baidu.com>

paddle/fluid/operators/run_program_op.h

@@ -102,74 +102,50 @@ static void CheckOutputVarStatus(const Variable &src_var,
 }
 
 static void VariableShare(const Variable &src_var, Variable *dst_var) {
-  // The previous check ensures that the variable type can only be LoDTensor
-  auto *lod_tensor = dst_var->GetMutable<LoDTensor>();
-  lod_tensor->ShareDataWith(src_var.Get<LoDTensor>());
-  lod_tensor->set_lod(src_var.Get<LoDTensor>().lod());
-}
-
-static void ShareVarsIntoScope(const std::vector<Variable *> &vars,
-                               const std::vector<std::string> &var_names,
-                               framework::Scope *scope) {
-  for (size_t i = 0; i < vars.size(); ++i) {
-    auto *var = scope->Var(var_names[i]);
-    CheckInputVarStatus(*vars[i], var_names[i]);
-    VariableShare(*vars[i], var);
-  }
-}
-
-static void VariableCopy(const Variable &src_var,
-                         const platform::Place &dst_place, Variable *dst_var) {
   // The previous check ensures that the variable type can only be LoDTensor or
-  // SelectedRows
+  // SelectedRows.
   if (src_var.IsType<LoDTensor>()) {
     auto *lod_tensor = dst_var->GetMutable<LoDTensor>();
-    TensorCopySync(src_var.Get<LoDTensor>(), dst_place, lod_tensor);
+    lod_tensor->ShareDataWith(src_var.Get<LoDTensor>());
     lod_tensor->set_lod(src_var.Get<LoDTensor>().lod());
   } else if (src_var.IsType<SelectedRows>()) {
     auto *selected_rows = dst_var->GetMutable<SelectedRows>();
-    TensorCopySync(src_var.Get<SelectedRows>().value(), dst_place,
-                   selected_rows->mutable_value());
+    selected_rows->mutable_value()->ShareDataWith(
+        src_var.Get<SelectedRows>().value());
     selected_rows->set_rows(src_var.Get<SelectedRows>().rows());
     selected_rows->set_height(src_var.Get<SelectedRows>().height());
   }
 }
 
-static void ShareVarsFromScope(const std::vector<Variable *> &vars,
+static void ShareVarsIntoScope(const std::vector<Variable *> &vars,
                                const std::vector<std::string> &var_names,
                                framework::Scope *scope) {
   for (size_t i = 0; i < vars.size(); ++i) {
-    auto *var = scope->FindVar(var_names[i]);
-    PADDLE_ENFORCE_NOT_NULL(
-        var, platform::errors::NotFound("The output variable %s is not in "
-                                        "RunProgram(Grad)Op(StaticModelRunner)'"
-                                        "s internal scope.",
-                                        var_names[i]));
-    CheckOutputVarStatus(*var, *vars[i], var_names[i]);
-    VariableShare(*var, vars[i]);
+    auto *var = scope->Var(var_names[i]);
+    CheckInputVarStatus(*vars[i], var_names[i]);
+    VariableShare(*vars[i], var);
   }
 }
 
-static void CopyVarsFromScope(const std::vector<Variable *> &vars,
-                              const std::vector<std::string> &var_names,
-                              const platform::Place &dst_place,
-                              framework::Scope *scope) {
+static void ShareVarsFromScope(const std::vector<Variable *> &vars,
+                               const std::vector<std::string> &var_names,
+                               framework::Scope *scope) {
   for (size_t i = 0; i < vars.size(); ++i) {
     if (var_names[i] == framework::kEmptyVarName) {
       VLOG(2) << "find variable name is " << framework::kEmptyVarName
               << ", skip it!";
       continue;
     }
-    auto *var = scope->FindVar(var_names[i]);
     // NOTE: Here skip not found var is dangerous, if a bug is caused here,
     // the result is grad calculation error, which will be very hidden!
+    auto *var = scope->FindVar(var_names[i]);
     PADDLE_ENFORCE_NOT_NULL(
         var, platform::errors::NotFound("The output variable %s is not in "
                                         "RunProgram(Grad)Op(StaticModelRunner)'"
                                         "s internal scope.",
                                         var_names[i]));
     CheckOutputVarStatus(*var, *vars[i], var_names[i]);
-    VariableCopy(*var, dst_place, vars[i]);
+    VariableShare(*var, vars[i]);
   }
 }
 
@@ -306,11 +282,9 @@ class RunProgramGradOpKernel : public framework::OpKernel<T> {
                                   end_op_index, /*create_local_scope=*/false,
                                   /*create_vars=*/true, /*keep_kids=*/false);
 
-    // Step 4. copy outputs
-    details::CopyVarsFromScope(input_grad_vars, input_grad_var_names,
-                               ctx.GetPlace(), &scope);
-    details::CopyVarsFromScope(param_grad_vars, param_grad_names,
-                               ctx.GetPlace(), &scope);
+    // Step 4. get outputs
+    details::ShareVarsFromScope(input_grad_vars, input_grad_var_names, &scope);
+    details::ShareVarsFromScope(param_grad_vars, param_grad_names, &scope);
   }
 };
 

python/paddle/fluid/data_feeder.py

@@ -76,7 +76,8 @@ def check_variable_and_dtype(input,
                              expected_dtype,
                              op_name,
                              extra_message=''):
-    check_type(input, input_name, Variable, op_name, extra_message)
+    check_type(input, input_name, (Variable, core.VarBase), op_name,
+               extra_message)
     check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message)
 
 

python/paddle/fluid/dygraph/base.py

@@ -61,6 +61,26 @@ def program_desc_tracing_guard(enable):
 _functional_dygraph_context_manager = None
 
 
+@signature_safe_contextmanager
+def param_guard(parameters):
+    # Note: parameters is a reference of self._parameters
+    if not framework.in_dygraph_mode() and parameters:
+        origin_parameters = parameters.copy()
+        for name, var_base in parameters.items():
+            if isinstance(var_base, core.VarBase):
+                new_var = framework.Parameter(
+                    var_base.block,
+                    var_base.shape,
+                    var_base.dtype,
+                    var_base.type,
+                    name=var_base.name)
+                parameters[name] = new_var
+        yield
+        parameters.update(origin_parameters)
+    else:
+        yield
+
+
 def enabled():
     """
     This function checks whether the program runs in dynamic graph mode or not.

python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py

@@ -14,8 +14,6 @@
 
 from __future__ import print_function
 
-import astor
-import copy
 # gast is a generic AST to represent Python2 and Python3's Abstract Syntax Tree(AST).
 # It provides a compatibility layer between the AST of various Python versions,
 # as produced by ast.parse from the standard ast module.
@@ -24,8 +22,6 @@ import gast
 import inspect
 import textwrap
 
-from paddle.fluid import unique_name
-
 from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer
 from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer
 from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfElseTransformer
@@ -34,14 +30,9 @@ from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import LoopTransfor
 from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import TensorShapeTransformer
 from paddle.fluid.dygraph.dygraph_to_static.call_transformer import CallTransformer
 
-from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper
-from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor
 from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func
-from paddle.fluid.dygraph.dygraph_to_static.utils import dygraph_class_to_static_api
 from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name
-from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api, is_dygraph_api, is_to_variable
-from paddle.fluid.dygraph.dygraph_to_static.utils import to_assign_node, to_static_ast, update_args_of_func
 
 __all__ = ['DygraphToStaticAst', 'convert_to_static']
 
@@ -142,6 +133,9 @@ def convert_to_static(dyfunc):
     Converts dygraph function into static function.
     """
     # Get AST from dygraph function
+    # Note: In Python2, it will raise OSError when inspect function
+    # with decorator directly and dyfunc.__wrapped__ holds the actual function.
+    dyfunc = getattr(dyfunc, '__wrapped__', dyfunc)
     raw_code = inspect.getsource(dyfunc)
     code = textwrap.dedent(raw_code)
     root = gast.parse(code)

python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py

@@ -17,9 +17,6 @@ from __future__ import print_function
 import gast
 
 from paddle.fluid import unique_name
-from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import NodeTestTransformer
-from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor
-from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code
 from paddle.fluid.dygraph.dygraph_to_static.utils import get_constant_variable_node
 from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list
 from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node

python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py

@@ -32,6 +32,15 @@ class CallTransformer(gast.NodeTransformer):
         self.wrapper_root = wrapper_root
         self.root = wrapper_root.node
 
+    def _is_builtin_call(self, node):
+        assert isinstance(node, gast.Call)
+        func_str = ast_to_source_code(node.func).strip()
+        try:
+            from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import is_builtin
+            return eval("is_builtin({})".format(func_str))
+        except Exception:
+            return False
+
     def transform(self):
         self.visit(self.root)
 
@@ -39,6 +48,10 @@ class CallTransformer(gast.NodeTransformer):
         self.generic_visit(node)
         if is_paddle_api(node):
             return node
+
+        if self._is_builtin_call(node):
+            return node
+
         func_str = ast_to_source_code(node.func).strip()
         new_func_str = "fluid.dygraph.dygraph_to_static.convert_call({})".format(
             func_str)

python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py

@@ -103,17 +103,8 @@ def convert_call(func):
             return func
         try:
             if func in func.__globals__.values():
-                if six.PY3:
-                    source_code = inspect.getsource(func)
-                    if any(decorator in source_code
-                           for decorator in DECORATOR_NAMES):
-                        converted_call = None
-                    else:
-                        converted_call = to_static_func(func)
-                        func_self = getattr(func, '__self__', None)
-                else:
-                    converted_call = to_static_func(func)
-                    func_self = getattr(func, '__self__', None)
+                converted_call = to_static_func(func)
+                func_self = getattr(func, '__self__', None)
         except AttributeError:
             # NOTE:
             # If func is not in __globals__, it does not need to be transformed
@@ -121,45 +112,25 @@ def convert_call(func):
             converted_call = None
         except (IOError, OSError):
             # NOTE:
-            # If func has beed decorated, its source code can not be get
+            # If func has been decorated, its source code can not be get
             # so that it can not be transformed to static function.
             converted_call = None
     elif inspect.ismethod(func):
         try:
-            if six.PY3:
-                source_code = inspect.getsource(func)
-                if any(decorator in source_code
-                       for decorator in DECORATOR_NAMES):
-                    converted_call = None
-                else:
-                    converted_call = to_static_func(func)
-                    func_self = getattr(func, '__self__', None)
-            else:
-                converted_call = to_static_func(func)
-                func_self = getattr(func, '__self__', None)
+            converted_call = to_static_func(func)
+            func_self = getattr(func, '__self__', None)
         except (IOError, OSError):
-            # NOTE: func may have beed decorated.
+            # NOTE: func may have been decorated.
             converted_call = None
 
     elif hasattr(func, '__class__') and hasattr(func.__class__, '__call__'):
         if hasattr(func, 'forward') and isinstance(func, Layer):
             try:
-                if six.PY3:
-                    source_code = inspect.getsource(func.forward)
-                    if any(decorator in source_code
-                           for decorator in DECORATOR_NAMES):
-                        converted_call = None
-                    else:
-                        forward_func = to_static_func(func.forward)
-                        setattr(func, 'forward', forward_func)
-                        func_self = func
-                else:
-                    forward_func = to_static_func(func.forward)
-                    setattr(func, 'forward', forward_func)
-                    func_self = func
-
+                forward_func = to_static_func(func.forward)
+                setattr(func, 'forward', forward_func)
+                func_self = func
             except Exception:
-                # NOTE: func.forward may have beed decorated.
+                # NOTE: func.forward may have been decorated.
                 func_self = None if func_self else func_self
             converted_call = func
         else:

python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py

@@ -22,15 +22,18 @@ from collections import defaultdict
 # as produced by ast.parse from the standard ast module.
 # See details in https://github.com/serge-sans-paille/gast/
 import gast
-import six
 from paddle.fluid import unique_name
 
+from paddle.fluid.dygraph.dygraph_to_static.utils import compare_with_none
+from paddle.fluid.dygraph.dygraph_to_static.utils import is_candidate_node
 from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api
 from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code
 from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node
 from paddle.fluid.dygraph.dygraph_to_static.utils import create_assign_node
+from paddle.fluid.dygraph.dygraph_to_static.utils import IsControlFlowVisitor
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor
-from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType
+from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper
+from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType
 
 TRUE_FUNC_PREFIX = 'true_fn'
 FALSE_FUNC_PREFIX = 'false_fn'
@@ -142,155 +145,25 @@ class IfElseTransformer(gast.NodeTransformer):
         return self.new_func_nodes
 
 
-def is_candidate_node(node):
-    """
-    Nodes with specified type will be dependent on tensor.
-    """
-    is_compare_node = isinstance(node,
-                                 (gast.Compare, gast.BoolOp, gast.UnaryOp))
-    # TODO(Aurelius84): `.numpy()` may be an customized function,
-    # and should consider a more elegant way to solve this problem.
-    has_numpy_attr = ".numpy()" in ast_to_source_code(node)
-    return is_compare_node or has_numpy_attr
-
-
-def compare_with_none(node):
-    """
-    Whether the comparator of `gast.Compare` node is `None`.
-    """
-    if isinstance(node, gast.Compare):
-        for child in [node.left, node.comparators]:
-            # node.comparators is a list.
-            if isinstance(child, list):
-                child = child[0]
-            if (isinstance(child, gast.Constant) and child.value is None) or (
-                    isinstance(child, gast.Name) and child.id == 'None'):
-                return True
-    return False
-
-
-class IsControlFlowVisitor(gast.NodeVisitor):
-    """
-    Judge whether the node.test from Dygraph code dependent on paddle Tensor.
-    If does, it should satisfy:
-        1. must involve at least one var whose type is Tensor.
-        2. the Tensor var should call `.numpy()[]` interface or Tensor.shape is [1].
-        3. involve Tensor.shape[i] and the shape[i] is unknown in compile time.
-    The following examples should not be considered as control_flow_if:
-        1. `if Tensor_var` or `if Tensor_var is None`
-        2. if Tensor.shape[i] is determined with fixed value (not -1 or None)
-
-    Note: pred in ConditionalBlock require variable, which means all vars should be Tensor
-          or transformed into Tensor, like fill_constant(shape=[1], dtype='int32', value=Tensor.shape[i]).
-
-    TODO: 1. need to deal with `tensor.shape[i]` which need to eval the data of shape[i],
-             because reshape_op may be called before this statement.
-    """
-
+class NodeTestTransformer(gast.NodeTransformer):
     def __init__(self,
                  ast_node,
-                 static_analysis_visitor=None,
-                 node_var_type_map=None):
-        assert isinstance(
-            ast_node, gast.AST
-        ), "Type of input node should be gast.AST, but received %s." % type(
-            ast_node)
-        self.ast_root = ast_node
-        if static_analysis_visitor is None:
-            static_analysis_visitor = StaticAnalysisVisitor(ast_node)
-        self.static_analysis_visitor = static_analysis_visitor
-        self.node_var_type_map = node_var_type_map
-
-        self.is_control_flow_num = 0
-        self._compare_node_tenor_set = set()
-
-    def transform(self):
-        node = self.ast_root
-        if is_candidate_node(node):
-            self.visit(node)
-        return self.is_control_flow_num > 0
-
-    def visit_BoolOp(self, node):
-        for i, child in enumerate(node.values):
-            if is_candidate_node(child):
-                self.visit(child)
-        return node
-
-    def visit_Compare(self, node):
-        # Ignores child node with `if x` or `if x is None`
-        # TODO(Aurelius84): `if tensor` will be supported in dygraph
-        # and should be considered as is_control_flow.
-        pre_control_flow_num = self.is_control_flow_num
-        if not compare_with_none(node):
-            self.generic_visit(node)
-            for child in gast.walk(node):
-                if isinstance(child, gast.Subscript):
-                    self._visit_Subscript(child)
-        if self.is_control_flow_num > pre_control_flow_num:
-            self._compare_node_tenor_set.add(node)
-        return node
-
-    def _visit_Subscript(self, node):
-        self.generic_visit(node)
-        if hasattr(node, 'value') and isinstance(node.value, gast.Call):
-            self._visit_Call(node.value)
-        return node
-
-    def _visit_Call(self, node):
-        assert isinstance(node, gast.Call)
-        if isinstance(node.func, gast.Attribute):
-            attr_node = node.func
-            if attr_node.attr == 'numpy':
-                self.is_control_flow_num += 1
-
-    def visit_Call(self, node):
-        self._visit_Call(node)
-        if is_paddle_api(node):
-            self.is_control_flow_num += 1
-        return node
-
-    def visit_Name(self, node):
-        if self._is_node_with_tensor(node, node.id):
-            self.is_control_flow_num += 1
-        return node
-
-    def visit_Constant(self, node):
-        if self._is_node_with_tensor(node, node.value):
-            self.is_control_flow_num += 1
-        return node
-
-    def _is_node_with_tensor(self, node, name_id):
-        tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES}
-        # Look up the node_var_type_map by name_id.
-        if self.node_var_type_map:
-            if name_id and isinstance(name_id, six.string_types):
-                var_type = self.node_var_type_map.get(name_id, None)
-                if var_type and var_type & tensor_types:
-                    return True
-        # if not found, look up the node_to_wrapper_map by node.
-        node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map(
-        )
-        wrapper_node = node_to_wrapper_map.get(node, None)
-        if wrapper_node is not None:
-            if wrapper_node.node_var_type & tensor_types:
-                return True
-
-        return False
-
-    def get_compare_nodes_with_tensor(self):
-        return self._compare_node_tenor_set
-
-
-class NodeTestTransformer(gast.NodeTransformer):
-    def __init__(self, ast_node, compare_nodes_with_tensor=None):
+                 compare_nodes_with_tensor=None,
+                 node_to_wrapper_map=None):
         if compare_nodes_with_tensor is None:
             compare_nodes_with_tensor = set()
         self.ast_root = ast_node
         self._compare_nodes_with_tensor = compare_nodes_with_tensor
+        if node_to_wrapper_map is None:
+            node_to_wrapper_map = {}
+        self.node_to_wrapper_map = node_to_wrapper_map
         self._new_assign_nodes = []
 
     def transform(self):
-        return self.visit(self.ast_root)
+        node = self.ast_root
+        if not is_candidate_node(node):
+            return self._create_cast_node(node)
+        return self.visit(node)
 
     def visit_Call(self, node):
         # Remove `numpy()` statement, like `Tensor.numpy()[i]` -> `Tensor[i]`
@@ -319,8 +192,11 @@ class NodeTestTransformer(gast.NodeTransformer):
     def visit_BoolOp(self, node):
         for i, child in enumerate(node.values):
             if not is_candidate_node(child):
-                node.values[i] = self._create_bool_node(child)
-                continue
+                node_wrapper = self.node_to_wrapper_map.get(child, None)
+                if node_wrapper and node_wrapper.node_var_type & NodeVarType.TENSOR_TYPES:
+                    node.values[i] = self._create_cast_node(child)
+                else:
+                    node.values[i] = self._create_bool_node(child)
         self.generic_visit(node)
         new_node = self._create_logic_node(node)
         return new_node
@@ -332,10 +208,19 @@ class NodeTestTransformer(gast.NodeTransformer):
         self.generic_visit(node)
         return node
 
+    def _create_cast_node(self, node):
+        template = "fluid.layers.cast(x={}, dtype='bool')"
+
+        return self._create_node_with_api_template(node, template)
+
     def _create_bool_node(self, node):
+        template = "fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool({}))"
+
+        return self._create_node_with_api_template(node, template)
+
+    def _create_node_with_api_template(self, node, template):
         node_code = ast_to_source_code(node)
-        new_node_str = "fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool({}))".format(
-            node_code)
+        new_node_str = template.format(node_code)
         # gast.parse return Module(body=[expr(value=...)])
         new_node = gast.parse(new_node_str).body[0].value
         bool_tensor_name = unique_name.generate(PLAIN_TENSOR_PREFIX)
@@ -395,7 +280,8 @@ class IfConditionVisitor(object):
         self.static_analysis_visitor = static_analysis_visitor
         self.visitor = IsControlFlowVisitor(node, static_analysis_visitor,
                                             node_var_type_map)
-        self.transformer = NodeTestTransformer(node)
+        self.transformer = NodeTestTransformer(
+            node, node_to_wrapper_map=self.visitor.node_to_wrapper_map)
         self.compare_nodes_with_tensor = set()
         self._is_control_flow_if = False
 

python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py

@@ -55,19 +55,22 @@ class ListTransformer(gast.NodeTransformer):
 
     def visit_If(self, node):
         self.generic_visit(node)
-        if is_control_flow_to_transform(node, self.scope_var_type_dict):
+        if is_control_flow_to_transform(node, self.static_analysis_visitor,
+                                        self.scope_var_type_dict):
             self._transform_list_append_in_control_flow(node)
         return node
 
     def visit_While(self, node):
         self.generic_visit(node)
-        if is_control_flow_to_transform(node, self.scope_var_type_dict):
+        if is_control_flow_to_transform(node, self.static_analysis_visitor,
+                                        self.scope_var_type_dict):
             self._transform_list_append_in_control_flow(node)
         return node
 
     def visit_For(self, node):
         self.generic_visit(node)
-        if is_control_flow_to_transform(node, self.scope_var_type_dict):
+        if is_control_flow_to_transform(node, self.static_analysis_visitor,
+                                        self.scope_var_type_dict):
             self._transform_list_append_in_control_flow(node)
         return node
 

python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py

@@ -26,6 +26,7 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code
 from paddle.fluid.dygraph.dygraph_to_static.utils import generate_name_node
 from paddle.fluid.dygraph.dygraph_to_static.utils import get_constant_variable_node
 from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name
+from paddle.fluid.dygraph.dygraph_to_static.utils import is_control_flow_to_transform
 from paddle.fluid.dygraph.dygraph_to_static.utils import RenameTransformer
 from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_static_variable_gast_node
 from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable_gast_node
@@ -150,8 +151,9 @@ class NameVisitor(gast.NodeVisitor):
         self.visit(root_node)
 
     def is_control_flow_loop(self, node):
-        # TODO: make a better condition
-        return True
+        need_transform = is_control_flow_to_transform(
+            node, self.static_analysis_visitor)
+        return need_transform
 
     def get_loop_var_names(self, node):
         assert isinstance(

python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import numpy as np
+from paddle.fluid import framework, backward, core
+from paddle.fluid.dygraph import layers
+from paddle.fluid.dygraph.base import switch_to_static_graph
+import paddle.compat as cpt
+
+
+class PartialProgramLayer(layers.Layer):
+    """
+    PartialProgramLayer wraps all the ops from layers decorated by `@declarative`
+    and execute them as a static subgraph.
+
+    .. note::
+        **1. This is a very low level API. Users should not use this API
+             directly. Please use `partial_program_from(concrete_program)`
+             to create it.
+        **2. LoDTensorArray is not currently supported in the output.
+
+    Args:
+        main_program(Program): The main program that contains ops need to be executed.
+        inputs(list[Variable]): The input list of the decorated function by `@declarative`.
+        outputs(list[Variable]): The output list of the decorated function by `@declarative`.
+        parameters(list[VarBase]|None): All trainable parameters included in the program. Default None.
+
+    Returns:
+        Layer: A Layer object that run all ops internally in static mode.
+    """
+
+    def __init__(self, main_program, inputs, outputs, parameters=None):
+        super(PartialProgramLayer, self).__init__()
+        self.inputs = inputs
+        self.outputs = outputs
+        self._params = parameters if parameters is not None else []
+        # Check all params from main program can be found in self._params:
+        # 1. parameter in self._params should be type `framework.ParamBase` which are created in dygraph.
+        # 2. parameter from transformed program shall be found in self._params.
+        #    Because they share same data with ParamBase of original dygraph.
+        self._check_params_all_inited(main_program)
+
+        self._infer_program = main_program
+        self._train_program = self._append_backward_desc()
+        # Switch infer or train by train() and eval()
+        self._trace_program = None
+        self._set_grad_type(self._params)
+        self._inner_scope = core.Scope()
+        # Set default mode to train
+        self.train()
+
+    @switch_to_static_graph
+    def _append_backward_desc(self):
+        program = self._infer_program.clone()
+        targets = []
+        for out in self.outputs:
+            if isinstance(out, framework.Variable):
+                targets.append(program.global_block().var(out.name))
+
+        if targets and self._params:
+            backward.gradients(targets=targets, inputs=[])
+
+        return program
+
+    def train(self):
+        # self.training is inherited from layers.Layer
+        self.training = True
+        self._trace_program = self._train_program
+
+    def eval(self):
+        self.training = False
+        self._trace_program = self._infer_program
+
+    def forward(self, inputs):
+        in_vars, out_vars, tmp_scope_vec = self._prepare(inputs)
+
+        framework._dygraph_tracer().trace_op(
+            type='run_program',
+            inputs={
+                'X': valid_vars(in_vars),
+                'Params': valid_vars(self._params)
+            },
+            outputs={'Out': valid_vars(out_vars),
+                     'OutScope': tmp_scope_vec},
+            attrs={
+                'global_block': self._trace_program.desc.block(0),
+                'start_op_index': 0,
+                'end_op_index': self._infer_program.desc.block(0).op_size(),
+                'is_test': not self.training
+            })
+
+        outs = out_vars
+        if len(outs) == 1:
+            outs = outs[0]
+        return outs
+
+    def _prepare(self, inputs):
+        """
+        Prepare inputs, outputs, attrs.
+        """
+        assert isinstance(inputs, (tuple, list))
+        # Convert variable into VarBase and feed in training data.
+        input_vars = []
+        for i, value in enumerate(inputs):
+            if isinstance(value, np.ndarray):
+                var = core.VarBase(
+                    value=value,
+                    name=self.inputs[i].desc.name(),
+                    persistable=False,
+                    place=framework._current_expected_place(),
+                    zero_copy=True)
+            elif isinstance(value, core.VarBase):
+                var = value
+                var.name = self.inputs[i].desc.name()
+            else:
+                continue
+            input_vars.append(var)
+        # Create VarBase to receive output data.
+        out_vars = []
+        for var in self.outputs:
+            if not isinstance(var, framework.Variable):
+                continue
+            var_desc = var.desc
+            var_base = core.VarBase(var_desc.dtype(),
+                                    var_desc.shape(),
+                                    var_desc.name(), var_desc.type(), False)
+            out_vars.append(var_base)
+
+        # Hold forward variables
+        tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [],
+                                     "program_out_scope",
+                                     core.VarDesc.VarType.STEP_SCOPES, True)
+
+        tmp_scope_vec.value().set_scope(self._inner_scope)
+
+        return input_vars, out_vars, tmp_scope_vec
+
+    def _set_grad_type(self, params):
+        # NOTE: if user set sparse gradient mode, the param's gradient
+        # will be SelectedRows, not LoDTensor. But tracer will just
+        # set param grad VarBase by forward VarBase(LoDTensor)
+        # If we don't change grad_var type here, RunProgramOp need
+        # transform SelectedRows to LoDTensor forcibly, it may not
+        # be user wanted result.
+        for param in params:
+            grad_name = param.name + core.grad_var_suffix()
+            grad_var = self._train_program.desc.block(0).find_var(
+                cpt.to_bytes(grad_name))
+            # NOTE: cannot find var desc maybe no problem, such as in batch_norm
+            if grad_var is None:
+                continue
+            param._set_grad_type(grad_var.type())
+
+    def _check_params_all_inited(self, main_program):
+        """
+        Check all params from main program are already initialized, see details as follows:
+            1. all parameters in self._params should be type `framework.ParamBase` which are created in dygraph.
+            2. all parameters from transformed program can be found in self._params.
+               Because they share same data with ParamBase of original dygraph.
+        """
+        if not isinstance(self._params, (list, tuple)):
+            raise TypeError(
+                "Type of self._params in PartialProgramLayer should be list or tuple, but received %s."
+                % type(self._params))
+
+        params_name_set = set()
+        for i, param in enumerate(self._params):
+            if not isinstance(param, framework.ParamBase):
+                raise TypeError(
+                    'Type of self._params[{}] in PartialProgramLayer should be framework.ParamBase, but received {}.'.
+                    format(i, type(param)))
+            params_name_set.add(param.name)
+
+        for block in main_program.blocks:
+            for name, var in block.vars.items():
+                if isinstance(var, framework.Parameter):
+                    if name not in params_name_set:
+                        raise ValueError(
+                            "\n\tWe don't support to define layer with parameters in the function "
+                            "decorated by `@declarative`.\n\tBecause that will re-defined parameters "
+                            "every time when you run the function.\n\t"
+                            "But we found parameter(%s) was created in the decorated function.\n\t"
+                            "Please define the layer with parameters in `__init__` function."
+                            % name)
+
+
+def valid_vars(vars):
+    """
+    Note: run_program_op.InferShape requires `X`/'Out' not be null.
+    But it's common in dy2static, fake varBase is created to handle the
+    problem.
+    """
+    if vars:
+        return vars
+    return [
+        core.VarBase(
+            value=[1],
+            name='Fake_var',
+            place=framework._current_expected_place())
+    ]
+
+
+def partial_program_from(concrete_program):
+    inputs = concrete_program.inputs
+    if inputs and isinstance(inputs[0], layers.Layer):
+        inputs = inputs[1:]
+
+    return PartialProgramLayer(concrete_program.main_program, inputs,
+                               concrete_program.outputs,
+                               concrete_program.parameters)

python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import gast
+
+from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType, StaticAnalysisVisitor
+
+
+class PrintTransformer(gast.NodeTransformer):
+    """
+    This class transforms python print function to fluid.layers.Print.
+    """
+
+    def __init__(self, wrapper_root):
+        assert isinstance(
+            wrapper_root, AstNodeWrapper
+        ), "Input non-AstNodeWrapper node for the initialization of PrintTransformer."
+        self.wrapper_root = wrapper_root
+        self.root = wrapper_root.node
+
+        self.static_analysis_visitor = StaticAnalysisVisitor(self.root)
+        self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map(
+        )
+
+    def transform(self):
+        self.visit(self.root)
+
+    # NOTE: deal with print in PY3
+    def visit_Call(self, node):
+        assert isinstance(node, gast.Call)
+        if isinstance(node.func, gast.Name) and node.func.id == 'print':
+            var = self._get_print_var(node)
+            return self._construct_print_node(var)
+        return node
+
+    # NOTE: deal with print in PY2
+    def visit_Print(self, node):
+        var = self._get_print_var(node)
+        print_call_node = self._construct_print_node(var)
+        return gast.Expr(value=print_call_node)
+
+    def _get_print_var(self, node):
+        if isinstance(node, gast.Call):
+            var_list = node.args
+        elif isinstance(node, gast.Print):
+            var_list = node.values
+            if isinstance(var_list[0], gast.Tuple):
+                var_list = var_list[0].elts
+        # TODO: support print multiple Var
+        assert len(var_list) == 1, "Now only support print one Variable."
+        return var_list[0]
+
+    def _construct_print_node(self, node):
+        if isinstance(node, gast.Name):
+            if self._is_tensor_node(node):
+                print_node = gast.Call(
+                    func=gast.parse('fluid.layers.Print').body[0].value,
+                    args=[node],
+                    keywords=[])
+                return print_node
+            else:
+                raise TypeError(
+                    "print object type error, only support print Variable now.")
+        else:
+            # TODO: may not only print with format
+            raise NotImplementedError(
+                "cannot transform print with format temporarily.")
+
+    def _is_tensor_node(self, node):
+        tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES}
+        wrapper_node = self.node_to_wrapper_map.get(node, None)
+        if wrapper_node is not None:
+            if wrapper_node.node_var_type & tensor_types:
+                return True
+        return False

python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py

@@ -54,6 +54,9 @@ class NodeVarType(object):
     # We use this enum value to denote the type return by a Paddle API
     PADDLE_RETURN_TYPES = 304
 
+    # If node.node_var_type in TENSOR_TYPES, it can be considered as tensor-dependent.
+    TENSOR_TYPES = {TENSOR, PADDLE_RETURN_TYPES}
+
     @staticmethod
     def binary_op_output_type(in_type1, in_type2):
         if in_type1 == in_type2:

python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py

@@ -15,15 +15,8 @@
 from __future__ import print_function
 
 import gast
-import astor
-import copy
-from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType, StaticAnalysisVisitor
-from paddle.fluid.dygraph.dygraph_to_static.utils import is_control_flow_to_transform
-from paddle.fluid import unique_name
-from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func
-from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api, is_dygraph_api, is_to_variable
-from paddle.fluid.dygraph.dygraph_to_static.utils import to_assign_node, to_static_ast, update_args_of_func
-from paddle.fluid.dygraph.dygraph_to_static.utils import dygraph_class_to_static_api, create_api_shape_node
+from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api
+from paddle.fluid.dygraph.dygraph_to_static.utils import create_api_shape_node
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor
 

python/paddle/fluid/dygraph/dygraph_to_static/utils.py

@@ -50,8 +50,9 @@ def is_api_in_module(node, module_prefix):
         #  source_file = inspect.getfile(dyfunc)
         #  import_statements = ImportVisitor(source_file).transform()
         #  import_str = "".join(import_statements)
-        import paddle.fluid as fluid
         import paddle
+        import paddle.fluid as fluid
+        import paddle.fluid.layers as layers
         from paddle.fluid.dygraph import to_variable
         import paddle.fluid.dygraph as dygraph
         return eval("_is_api_in_module_helper({}, '{}')".format(func_str,
@@ -88,29 +89,19 @@ def is_numpy_api(node):
         return False
 
 
-def is_control_flow_to_transform(node, var_name_to_type):
+def is_control_flow_to_transform(node,
+                                 static_analysis_visitor=None,
+                                 var_name_to_type=None):
     """
-    Determines whether the node is a Paddle control flow statement which needs to
-    transform into a static graph control flow statement.
+    Determines whether the node is a PaddlePaddle control flow statement which needs to
+    be transformed into a static graph control flow statement.
     """
     assert isinstance(node, gast.AST), \
         "The type of input node must be gast.AST, but received %s." % type(node)
-
-    if isinstance(node, gast.If):
-        from .ifelse_transformer import IfConditionVisitor
-        if_visitor = IfConditionVisitor(
-            node.test, node_var_type_map=var_name_to_type)
-        return if_visitor.is_control_flow()
-
-    if isinstance(node, gast.For):
-        # TODO: make a better condition
-        return True
-
-    if isinstance(node, gast.While):
-        # TODO: make a better condition
-        return True
-
-    return False
+    visitor = IsControlFlowVisitor(
+        node, static_analysis_visitor, node_var_type_map=var_name_to_type)
+    need_to_transform = visitor.transform()
+    return need_to_transform
 
 
 def _delete_keywords_from(node):
@@ -415,3 +406,208 @@ def ast_to_source_code(ast_node):
         ast_node = gast.gast_to_ast(ast_node)
     source_code = astor.to_source(ast_node)
     return source_code
+
+
+def is_candidate_node(node):
+    """
+    Nodes with specified type will be dependent on tensor.
+    """
+    is_compare_node = isinstance(node, (gast.Compare, gast.BoolOp, gast.UnaryOp,
+                                        gast.For, gast.If, gast.While))
+    # TODO(Aurelius84): `.numpy()` may be an customized function,
+    # and should consider a more elegant way to solve this problem.
+    has_numpy_attr = ".numpy()" in ast_to_source_code(node)
+    return is_compare_node or has_numpy_attr
+
+
+def compare_with_none(node):
+    """
+    Whether the comparator of `gast.Compare` node is `None`.
+    """
+    if isinstance(node, gast.Compare):
+        for child in [node.left, node.comparators]:
+            # node.comparators is a list.
+            if isinstance(child, list):
+                child = child[0]
+            if (isinstance(child, gast.Constant) and child.value is None) or (
+                    isinstance(child, gast.Name) and child.id == 'None'):
+                return True
+    return False
+
+
+class IsControlFlowVisitor(gast.NodeVisitor):
+    """
+    Judge whether the ast_node of control flow from Dygraph code dependent on paddle Tensor.
+    `ast_node` can be gast.If, gast.For, gast.While, gast.If.test(gast.Compare, gast.BoolOp, gast.UnaryOp).
+
+    If returns True,
+    gast.If.test must meet at least one of the following requirements:
+        1. involves at least one var whose type is Tensor.
+        2. the Tensor var calls `.numpy()[]` interface or Tensor.shape is [1].
+        3. involves Tensor.shape[i] and the shape[i] is unknown in compile time.
+    gast.While must meet at least one of the requirements 1 to 5:
+        4. has `break` statement.
+        5. has `continue` statement.
+    gast.For must meet at least one of the requirements 4 to 6:
+        6. calls `range` function in `for` statement and the argument of range is Tensor.
+        TODO: Support non-range case
+
+    The following examples should not be considered as control_flow_if:
+        1. `if Tensor_var` or `if Tensor_var is None`
+        2. if Tensor.shape[i] is determined with fixed value (not -1 or None)
+
+    Note: pred in ConditionalBlock require variable, which means all vars should be Tensor
+          or transformed into Tensor, like fill_constant(shape=[1], dtype='int32', value=Tensor.shape[i]).
+
+    TODO: 1. need to deal with `tensor.shape[i]` which need to eval the data of shape[i],
+             because reshape_op may be called before this statement.
+    """
+
+    def __init__(self,
+                 ast_node,
+                 static_analysis_visitor=None,
+                 node_var_type_map=None):
+        assert isinstance(
+            ast_node, gast.AST
+        ), "Type of input node should be gast.AST, but received %s." % type(
+            ast_node)
+        self.ast_root = ast_node
+        if static_analysis_visitor is None:
+            from .static_analysis import StaticAnalysisVisitor
+            static_analysis_visitor = StaticAnalysisVisitor(ast_node)
+        self.static_analysis_visitor = static_analysis_visitor
+        self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map(
+        )
+        self.node_var_type_map = node_var_type_map
+
+        self.is_control_flow_num = 0
+        self._compare_node_tenor_set = set()
+
+    def transform(self):
+        node = self.ast_root
+        if isinstance(node, gast.If):
+            self._visit_If(node)
+        elif isinstance(node, gast.For):
+            self._visit_For(node)
+        elif isinstance(node, gast.While):
+            self._visit_While(node)
+        else:
+            self.visit(node)
+        return self.is_control_flow_num > 0
+
+    def _visit_If(self, node):
+        assert isinstance(node, gast.If)
+        self.visit(node.test)
+        return
+
+    def _visit_For(self, node):
+        assert isinstance(node, gast.For)
+        # TODO
+        # self.is_control_flow_num += 1
+        if not isinstance(node.iter, gast.Call):
+            return
+        if not isinstance(node.iter.func, gast.Name):
+            return
+        if node.iter.func.id != "range":
+            return
+        for arg in node.iter.args:
+            self.visit(arg)
+
+        for child_node in gast.walk(node):
+            if isinstance(child_node, (gast.Continue, gast.Break)):
+                self._visit_break_continue(child_node)
+        return
+
+    def _visit_While(self, node):
+        assert isinstance(node, gast.While)
+        test = node.test
+        self.generic_visit(test)
+        for child_node in gast.walk(node):
+            if isinstance(child_node, (gast.Continue, gast.Break)):
+                self._visit_break_continue(child_node)
+        return
+
+    def _visit_break_continue(self, node):
+        assert isinstance(node, (gast.Break, gast.Continue))
+        wrapper_node = self.node_to_wrapper_map.get(node)
+        if not wrapper_node:
+            # Transformed node is not in node_to_wrapper_map
+            return
+
+        while wrapper_node.parent:
+            parent_node = wrapper_node.parent.node
+            if isinstance(parent_node, (gast.For, gast.While)):
+                if parent_node is self.ast_root:
+                    self.is_control_flow_num += 1
+                    return
+                else:
+                    return
+
+            wrapper_node = wrapper_node.parent
+
+        return
+
+    def visit_BoolOp(self, node):
+        for i, child in enumerate(node.values):
+            self.visit(child)
+        return node
+
+    def visit_Compare(self, node):
+        pre_control_flow_num = self.is_control_flow_num
+        if not compare_with_none(node):
+            self.generic_visit(node)
+            for child in gast.walk(node):
+                if isinstance(child, gast.Subscript):
+                    self._visit_Subscript(child)
+        if self.is_control_flow_num > pre_control_flow_num:
+            self._compare_node_tenor_set.add(node)
+        return node
+
+    def _visit_Subscript(self, node):
+        self.generic_visit(node)
+        if hasattr(node, 'value') and isinstance(node.value, gast.Call):
+            self._visit_Call(node.value)
+        return node
+
+    def _visit_Call(self, node):
+        assert isinstance(node, gast.Call)
+        if isinstance(node.func, gast.Attribute):
+            attr_node = node.func
+            if attr_node.attr == 'numpy':
+                self.is_control_flow_num += 1
+
+    def visit_Call(self, node):
+        self._visit_Call(node)
+        if is_paddle_api(node):
+            self.is_control_flow_num += 1
+        return node
+
+    def visit_Name(self, node):
+        if self._is_node_with_tensor(node, node.id):
+            self.is_control_flow_num += 1
+        return node
+
+    def visit_Constant(self, node):
+        if self._is_node_with_tensor(node, node.value):
+            self.is_control_flow_num += 1
+        return node
+
+    def _is_node_with_tensor(self, node, name_id):
+        from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType
+
+        # Look up the node_var_type_map by name_id.
+        if self.node_var_type_map:
+            if name_id and isinstance(name_id, six.string_types):
+                var_type = self.node_var_type_map.get(name_id, None)
+                if var_type and var_type & NodeVarType.TENSOR_TYPES:
+                    return True
+        # if not found, look up the node_to_wrapper_map by node.
+        wrapper_node = self.node_to_wrapper_map.get(node, None)
+        if wrapper_node is not None:
+            if wrapper_node.node_var_type & NodeVarType.TENSOR_TYPES:
+                return True
+
+        return False
+
+    def get_compare_nodes_with_tensor(self):
+        return self._compare_node_tenor_set

python/paddle/fluid/dygraph/jit.py

@@ -17,7 +17,6 @@ from __future__ import print_function
 __all__ = ['TracedLayer', 'declarative', 'dygraph_to_static_func']
 
 import logging
-
 from paddle.fluid import core
 from paddle.fluid.compiler import CompiledProgram
 from paddle.fluid.dygraph.base import program_desc_tracing_guard, switch_to_static_graph
@@ -156,13 +155,11 @@ def _declarative_(dygraph_func):
 
     def __impl__(*args, **kwargs):
         program_translator = ProgramTranslator()
-        if in_dygraph_mode() or not program_translator.enable_declarative:
+        if not program_translator.enable_declarative:
             logger.info(
-                "The decorator 'declarative' doesn't work in dygraph "
-                "mode or set ProgramTranslator.enable to False. We will "
-                "just return dygraph output.")
+                "The decorator 'declarative' doesn't work when setting ProgramTranslator.enable=False. "
+                "We will just return dygraph output.")
             return dygraph_func(*args, **kwargs)
-        program_translator = ProgramTranslator()
         return program_translator.get_output(dygraph_func, *args, **kwargs)
 
     return __impl__
@@ -228,6 +225,7 @@ class TracedLayer(object):
         self._program = program
         self._feed_names = feed_names
         self._fetch_names = fetch_names
+        self._params = parameters
 
         self._place = _current_expected_place()
 

python/paddle/fluid/dygraph/layers.py

@@ -23,7 +23,7 @@ from . import parallel_helper
 from .. import unique_name
 from paddle.fluid import core
 from .layer_object_helper import LayerObjectHelper
-from .base import program_desc_tracing_guard
+from .base import program_desc_tracing_guard, param_guard
 from paddle.fluid import framework
 from ..param_attr import ParamAttr
 import copy
@@ -457,7 +457,8 @@ class Layer(core.Layer):
                         self._parameters.values())
             self._built = True
 
-        outputs = self.forward(*inputs, **kwargs)
+        with param_guard(self._parameters):
+            outputs = self.forward(*inputs, **kwargs)
 
         for forward_post_hook in self._forward_post_hooks.values():
             hook_result = forward_post_hook(self, inputs, outputs)

python/paddle/fluid/framework.py

@@ -1915,7 +1915,7 @@ class Operator(object):
                                 in_arg_names.append(arg)
                             elif isinstance(arg, six.binary_type):
                                 in_arg_names.append(arg.decode())
-                            elif isinstance(arg, Variable):
+                            elif isinstance(arg, (Variable, core.VarBase)):
                                 in_arg_names.append(cpt.to_text(arg.name))
                             else:
                                 raise TypeError(

python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py

@@ -15,7 +15,6 @@
 from __future__ import print_function
 
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
 
 
 def add_fn(x):
@@ -142,7 +141,6 @@ class NetWithControlFlowIf(fluid.dygraph.Layer):
         self.alpha = 10.
         self.constant_vars = {}
 
-    @dygraph_to_static_func
     def forward(self, input):
         hidden_dim = input.shape[-1]
         if hidden_dim != self.hidden_dim:
@@ -250,3 +248,31 @@ def if_with_class_var(x, y=None):
     else:
         x = x - foo.b
     return x
+
+
+def if_tensor_case(x):
+    x = fluid.dygraph.to_variable(x)
+
+    mean = fluid.layers.mean(x)
+    # It is equivalent to `if mean != 0`
+    if mean:
+        for i in range(0, 10):
+            if i > 5:
+                x += 1
+                break
+            x += 1
+    else:
+        for i in range(0, 37):
+            x += 1
+            break
+            x += i
+
+    # join `and`/`or`
+    if fluid.layers.mean(x) + 1 and mean > 1 and x is not None or 2 > 1:
+        x -= 1
+
+    # `not` statement
+    if not (x[0][0] and (mean * x)[0][0]):
+        x += 1
+
+    return x

python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py

@@ -17,7 +17,7 @@ from __future__ import print_function
 import unittest
 import numpy as np
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph.jit import declarative
 
 SEED = 2020
 np.random.seed(SEED)
@@ -160,13 +160,9 @@ class TestContinueInFor(unittest.TestCase):
             return res.numpy()
 
     def run_static_mode(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            res = dygraph_to_static_func(self.dygraph_func)(self.input)
-        exe = fluid.Executor(self.place)
-        static_res = exe.run(main_program, fetch_list=[res])
-
-        return static_res[0]
+        with fluid.dygraph.guard():
+            res = declarative(self.dygraph_func)(self.input)
+            return res.numpy()
 
     def test_transformed_static_result(self):
         static_res = self.run_static_mode()

python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py

@@ -36,8 +36,7 @@ class TestCacheProgram(unittest.TestCase):
     def test_cache(self):
         prev_ops, cur_ops = Counter(), Counter()
         prev_out, cur_out = None, None
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
+        with fluid.dygraph.guard(fluid.CPUPlace()):
             static_net = self.dygraph_class()
             for batch_id in range(self.batch_num):
                 out = static_net(self.data)
@@ -51,9 +50,9 @@ class TestCacheProgram(unittest.TestCase):
                 ])
                 if batch_id > 0:
                     prev_out_numpy = prev_out[0].numpy() if isinstance(
-                        prev_out, tuple) else prev_out.numpy()
+                        prev_out, (tuple, list)) else prev_out.numpy()
                     cur_out_numpy = cur_out[0].numpy() if isinstance(
-                        cur_out, tuple) else cur_out.numpy()
+                        cur_out, (tuple, list)) else cur_out.numpy()
                     self.assertTrue(
                         np.allclose(prev_out_numpy, cur_out_numpy),
                         msg='Output in previous batch is {}\n Output in current batch is \n{}'
@@ -75,29 +74,23 @@ class TestCacheProgramWithOptimizer(unittest.TestCase):
         self.batch_num = 5
 
     def train_static(self):
-        main_program = fluid.Program()
-        loss_data = []
-        with fluid.program_guard(main_program):
-            static_net = self.dygraph_class()
-            adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001)
-            # set optimizer
-            program_translator = ProgramTranslator()
-            program_translator.set_optimizer(adam, index_of_loss=1)
+        return self.train(to_static=True)
 
-            for batch_id in range(self.batch_num):
-                pred, avg_loss = static_net(self.data)
-                loss_data.append(np.array(avg_loss.numpy()))
+    def train_dygraph(self):
+        return self.train(to_static=False)
 
-        return loss_data
+    def train(self, to_static=False):
+        prog_trans = ProgramTranslator()
+        prog_trans.enable(to_static)
 
-    def train_dygraph(self):
         with fluid.dygraph.guard(fluid.CPUPlace()):
             dygraph_net = self.dygraph_class()
             adam = fluid.optimizer.AdamOptimizer(
                 learning_rate=0.001, parameter_list=dygraph_net.parameters())
             loss_data = []
             for batch_id in range(self.batch_num):
-                pred, avg_loss = dygraph_net(self.data)
+                input = fluid.dygraph.to_variable(self.data)
+                pred, avg_loss = dygraph_net(input)
 
                 loss_data.append(avg_loss.numpy())
                 avg_loss.backward()
@@ -114,20 +107,6 @@ class TestCacheProgramWithOptimizer(unittest.TestCase):
             msg='dygraph is {}\n static_res is \n{}'.format(dygraph_loss,
                                                             static_loss))
 
-    def test_exception(self):
-        main_program = fluid.Program()
-        loss_data = []
-        with fluid.program_guard(main_program):
-            static_net = self.dygraph_class()
-            adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001)
-            # set optimizer
-            program_translator = ProgramTranslator()
-
-            with self.assertRaisesRegexp(ValueError, "has already been set"):
-                for batch_id in range(self.batch_num):
-                    program_translator.set_optimizer(adam, index_of_loss=1)
-                    static_net(self.data)
-
 
 def simple_func(x):
     inputs = fluid.dygraph.to_variable(x)
@@ -156,7 +135,6 @@ def sum_even_util_limit(max_len, limit):
     return ret_sum
 
 
-@declarative
 def sum_under_while(limit):
     i = fluid.dygraph.to_variable(np.zeros((1)).astype('int32'))
     ret_sum = fluid.dygraph.to_variable(np.zeros((1)).astype('int32'))
@@ -168,11 +146,12 @@ def sum_under_while(limit):
 
 class TestToOutputWithCache(unittest.TestCase):
     def test_output(self):
-        ret = sum_even_util_limit(80, 10)
-        self.assertEqual(ret.numpy(), 30)
+        with fluid.dygraph.guard():
+            ret = sum_even_util_limit(80, 10)
+            self.assertEqual(ret.numpy(), 30)
 
-        ret = sum_under_while(100)
-        self.assertEqual(ret.numpy(), 5050)
+            ret = declarative(sum_under_while)(100)
+            self.assertEqual(ret.numpy(), 5050)
 
 
 if __name__ == '__main__':

python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py

@@ -19,7 +19,8 @@ import numpy as np
 import unittest
 
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator
 
 PLACE = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace(
 )
@@ -75,7 +76,7 @@ class MainNetWithDict(fluid.dygraph.Layer):
         self.output_size = output_size
         self.sub_net = SubNetWithDict(hidden_size, output_size)
 
-    @dygraph_to_static_func
+    @declarative
     def forward(self, input, max_len=4):
         input = fluid.dygraph.to_variable(input)
         cache = {
@@ -121,17 +122,14 @@ class TestNetWithDict(unittest.TestCase):
         self.batch_size = self.x.shape[0]
 
     def _run_static(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            net = MainNetWithDict(batch_size=self.batch_size)
-            # Transform into static graph
-            out = net(self.x)
-            exe = fluid.Executor(PLACE)
-            exe.run(fluid.default_startup_program())
-            ret = exe.run(main_program, fetch_list=out)
-            return ret[0]
+        return self.train(to_static=True)
 
     def _run_dygraph(self):
+        return self.train(to_static=False)
+
+    def train(self, to_static=False):
+        prog_trans = ProgramTranslator()
+        prog_trans.enable(to_static)
         with fluid.dygraph.guard(PLACE):
             net = MainNetWithDict(batch_size=self.batch_size)
             ret = net(self.x)

python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py

@@ -14,12 +14,12 @@
 
 from __future__ import print_function
 
-from paddle.fluid.dygraph.jit import declarative
-
 import numpy as np
 import unittest
 
 import paddle.fluid as fluid
+from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
 
 SEED = 2020
 
@@ -32,22 +32,20 @@ class Pool2D(fluid.dygraph.Layer):
 
     @declarative
     def forward(self, x):
-        inputs = fluid.dygraph.to_variable(x)
-
         # Add func `get_result` for testing arg_name_to_idx in ast transformation.
         def get_result(x):
             return self.pool2d(x)
 
-        pre = get_result(inputs)
+        pre = get_result(x)
         return pre
 
 
 class Linear(fluid.dygraph.Layer):
-    def __init__(self):
+    def __init__(self, input_dim=10, output_dim=5):
         super(Linear, self).__init__()
         self.fc = fluid.dygraph.Linear(
-            input_dim=10,
-            output_dim=5,
+            input_dim,
+            output_dim,
             act='relu',
             param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(
                 value=0.99)),
@@ -56,8 +54,7 @@ class Linear(fluid.dygraph.Layer):
 
     @declarative
     def forward(self, x):
-        inputs = fluid.dygraph.to_variable(x)
-        pre = self.fc(inputs)
+        pre = self.fc(x)
         loss = fluid.layers.mean(pre)
         return pre, loss
 
@@ -67,28 +64,28 @@ class TestPool2D(unittest.TestCase):
         self.dygraph_class = Pool2D
         self.data = np.random.random((1, 2, 4, 4)).astype('float32')
 
-    def run_dygraph_mode(self):
+    def train(self, to_static=False):
+        program_translator = ProgramTranslator()
+        program_translator.enable(to_static)
+
         with fluid.dygraph.guard():
             dy_layer = self.dygraph_class()
-            prediction = dy_layer(x=self.data)
+            x = fluid.dygraph.to_variable(self.data)
+            prediction = dy_layer(x)
             if isinstance(prediction, (list, tuple)):
                 prediction = prediction[0]
 
             return prediction.numpy()
 
-    def run_static_mode(self):
-        startup_prog = fluid.Program()
-        main_prog = fluid.Program()
-        with fluid.program_guard(main_prog, startup_prog):
-            dy_layer = self.dygraph_class()
-            out = dy_layer(x=self.data)
-            if isinstance(out, tuple):
-                return out[0].numpy()
-            return out.numpy()
-
-    def test_static_output(self):
-        dygraph_res = self.run_dygraph_mode()
-        static_res = self.run_static_mode()
+    def train_static(self):
+        return self.train(to_static=True)
+
+    def train_dygraph(self):
+        return self.train(to_static=False)
+
+    def test_declarative(self):
+        dygraph_res = self.train_dygraph()
+        static_res = self.train_static()
 
         self.assertTrue(
             np.allclose(dygraph_res, static_res),

python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py

@@ -17,8 +17,7 @@ from __future__ import print_function
 import numpy as np
 import paddle.fluid as fluid
 import unittest
-
-from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph import declarative
 
 
 @fluid.dygraph.declarative
@@ -31,7 +30,7 @@ def dygraph_decorated_func(x):
     return x_v
 
 
-@fluid.dygraph.jit.declarative
+@fluid.dygraph.declarative
 def jit_decorated_func(x):
     x = fluid.dygraph.to_variable(x)
     if fluid.layers.mean(x) > 0:
@@ -62,18 +61,14 @@ class TestFullNameDecorator(unittest.TestCase):
     def test_run_success(self):
         x = np.ones([1, 2]).astype("float32")
         answer = np.zeros([1, 2]).astype("float32")
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
+        with fluid.dygraph.guard():
             self.assertTrue(
                 np.allclose(dygraph_decorated_func(x).numpy(), answer))
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
             self.assertTrue(np.allclose(jit_decorated_func(x).numpy(), answer))
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
             self.assertTrue(
                 np.allclose(decorated_call_decorated(x).numpy(), answer))
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
             with self.assertRaises(NotImplementedError):
                 DoubleDecorated().double_decorated_func1(x)
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
             with self.assertRaises(NotImplementedError):
                 DoubleDecorated().double_decorated_func2(x)
 

python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py

@@ -15,10 +15,10 @@
 from __future__ import print_function
 
 import numpy as np
-import paddle.fluid as fluid
 import unittest
 
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator
 
 from ifelse_simple_func import *
 
@@ -41,19 +41,16 @@ class TestDygraphIfElse(unittest.TestCase):
         self.dyfunc = dyfunc_with_if_else
 
     def _run_static(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            x_v = fluid.layers.assign(self.x)
-            # Transform into static graph
-            out = dygraph_to_static_func(self.dyfunc)(x_v)
-            exe = fluid.Executor(place)
-            ret = exe.run(main_program, fetch_list=out)
-            return ret
+        return self._run_dygraph(to_static=True)
+
+    def _run_dygraph(self, to_static=False):
 
-    def _run_dygraph(self):
         with fluid.dygraph.guard(place):
             x_v = fluid.dygraph.to_variable(self.x)
-            ret = self.dyfunc(x_v)
+            if to_static:
+                ret = declarative(self.dyfunc)(x_v)
+            else:
+                ret = self.dyfunc(x_v)
             return ret.numpy()
 
     def test_ast_to_func(self):
@@ -176,6 +173,12 @@ class TestDygraphIfElseWithClassVar(TestDygraphIfElse):
         self.dyfunc = if_with_class_var
 
 
+class TestDygraphIfTensor(TestDygraphIfElse):
+    def setUp(self):
+        self.x = np.random.random([10, 16]).astype('float32')
+        self.dyfunc = if_tensor_case
+
+
 class TestDygraphIfElseNet(unittest.TestCase):
     """
     TestCase for the transformation from control flow `if/else`
@@ -187,18 +190,15 @@ class TestDygraphIfElseNet(unittest.TestCase):
         self.Net = NetWithControlFlowIf
 
     def _run_static(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            net = self.Net()
-            x_v = fluid.layers.assign(self.x)
-            # Transform into static graph
-            out = net(x_v)
-            exe = fluid.Executor(place)
-            exe.run(fluid.default_startup_program())
-            ret = exe.run(main_program, fetch_list=out)
-            return ret[0]
+        return self._run(to_static=True)
 
     def _run_dygraph(self):
+        return self._run(to_static=False)
+
+    def _run(self, to_static=False):
+        prog_trans = ProgramTranslator()
+        prog_trans.enable(to_static)
+
         with fluid.dygraph.guard(place):
             net = self.Net()
             x_v = fluid.dygraph.to_variable(self.x)
@@ -234,7 +234,7 @@ class TestAst2FuncWithExternalFunc(TestDygraphIfElse):
 
 
 class NetWithExternalFunc(fluid.dygraph.Layer):
-    @dygraph_to_static_func
+    @declarative
     def forward(self, x, label=None):
         if fluid.layers.mean(x) < 0:
             x_v = x - 1

python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py

@@ -19,9 +19,9 @@ import textwrap
 import gast
 from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import get_name_ids
 from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfConditionVisitor
-from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IsControlFlowVisitor
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor
 from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType
+from paddle.fluid.dygraph.dygraph_to_static.utils import IsControlFlowVisitor
 
 
 class TestGetNameIds(unittest.TestCase):
@@ -155,7 +155,18 @@ class TestIsControlFlowIf(unittest.TestCase):
         self.check_false_case("fluid.layers.sum(x).numpy() != None")
 
     def test_is_None4(self):
-        self.check_false_case("fluid.layers.sum(x) and 2>1")
+        node = gast.parse("fluid.layers.sum(x) and 2>1")
+        node_test = node.body[0].value
+
+        if_visitor = IfConditionVisitor(node_test)
+        self.assertTrue(if_visitor.is_control_flow())
+        # Transformation result:
+        # bool_tensor_0 = fluid.layers.cast(x=fluid.layers.sum(x), dtype='bool')
+        # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1))
+        # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=bool_tensor_1)
+
+        new_node, assign_nodes = if_visitor.transform()
+        self.assertTrue(len(assign_nodes) == 3)
 
     def test_if(self):
         node = gast.parse("x.numpy()[1] > 1")
@@ -253,34 +264,38 @@ class TestIsControlFlowIf(unittest.TestCase):
         self.assertTrue(len(assign_nodes) == 0)
 
     def test_paddle_api_with_andOr(self):
-        code = """
+        code_or = """
             def foo(x):
                 if 2 > 1 and fluid.layers.shape(x)[0] > 16 or x is not None :
                     x = x + 1
                 return x
         """
-        code = """
+
+        code_and = """
             def foo(x):
                 if 2 > 1 and fluid.layers.shape(x)[0] > 16 and x is not None :
                     x = x + 1
                 return x
         """
-        code = textwrap.dedent(code)
-        node = gast.parse(code)
-        static_analysis_visitor = StaticAnalysisVisitor(node)
-        test_node = node.body[0].body[0].test
-        if_visitor = IfConditionVisitor(test_node, static_analysis_visitor)
-        self.assertTrue(if_visitor.is_control_flow())
-
-        new_node, assign_nodes = if_visitor.transform()
-        # Tranformation result:
-        # bool_tensor_0 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1))
-        # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(x is not None))
-        # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=fluid.layers.shape(x)[0] > 16)
-        # logic_and_1 = fluid.layers.logical_and(x=logic_and_0, y=bool_tensor_1)
-
-        self.assertTrue(isinstance(new_node, gast.Name))
-        self.assertTrue(len(assign_nodes) == 4)
+        for code in [code_or, code_and]:
+            code = textwrap.dedent(code)
+            node = gast.parse(code)
+            static_analysis_visitor = StaticAnalysisVisitor(node)
+            test_node = node.body[0].body[0].test
+            if_visitor = IfConditionVisitor(test_node, static_analysis_visitor)
+            self.assertTrue(if_visitor.is_control_flow())
+
+            new_node, assign_nodes = if_visitor.transform()
+            # Transformation result:
+            # bool_tensor_0 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1))
+            # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(x is not None))
+            # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=fluid.layers.shape(x)[0] > 16)
+
+            # logic_and_1 = fluid.layers.logical_and(x=logic_and_0, y=bool_tensor_1)  for code_and
+            # logic_or_0= fluid.layers.logical_or(x=logic_and_0, y=bool_tensor_1)  for code_and
+
+            self.assertTrue(isinstance(new_node, gast.Name))
+            self.assertTrue(len(assign_nodes) == 4)
 
     def test_with_node_var_type_map(self):
         node = gast.parse("x > 1")

python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py

@@ -47,6 +47,10 @@ def test_list_in_if(x):
 
 def test_list_in_for_loop(x, iter_num):
     x = fluid.dygraph.to_variable(x)
+    # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor
+    iter_num = fluid.layers.fill_constant(
+        shape=[1], value=iter_num, dtype="int32"
+    )  # TODO(liym27): Delete it if the type of parameter iter_num can be resolved
     a = []
     for i in range(iter_num):
         a.append(x)
@@ -56,6 +60,10 @@ def test_list_in_for_loop(x, iter_num):
 def test_list_in_for_loop_with_concat(x, iter_num):
     x = fluid.dygraph.to_variable(x)
     a = []
+    # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor
+    iter_num = fluid.layers.fill_constant(
+        shape=[1], value=iter_num, dtype="int32"
+    )  # TODO(liym27): Delete it if the type of parameter iter_num can be resolved
     for i in range(iter_num):
         a.append(x)
     a = fluid.layers.concat(a, axis=0)

python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py

@@ -20,8 +20,8 @@ import numpy as np
 import paddle.fluid as fluid
 import unittest
 
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
 from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import NameVisitor
+from paddle.fluid.dygraph.jit import declarative
 
 SEED = 2020
 np.random.seed(SEED)
@@ -29,6 +29,9 @@ np.random.seed(SEED)
 
 def while_loop_dyfunc(x):
     i = fluid.dygraph.to_variable(x)
+    # Use `to_variable` so that static analysis can analyze the type of X is Tensor
+    x = fluid.dygraph.to_variable(
+        x)  # TODO(liym27): Delete it if the type of parameter x can be resolved
     while x < 10:
         i = i + x
         x = x + 1
@@ -37,6 +40,9 @@ def while_loop_dyfunc(x):
 
 def while_loop_dyfun_with_conflict_var(x):
     i = fluid.dygraph.to_variable(x)
+    # Use `to_variable` so that static analysis can analyze the type of X is Tensor
+    x = fluid.dygraph.to_variable(
+        x)  # TODO(liym27): Delete it if the type of parameter x can be resolved
 
     def relu(y):
         # 'y' is not visible outside the scope.
@@ -56,6 +62,9 @@ def while_loop_dyfunc_with_none(x):
     i = fluid.dygraph.to_variable(x)\
         if x is not None \
         else fluid.dygraph.to_variable(x+1)
+    # Use `to_variable` so that static analysis can analyze the type of X is Tensor
+    x = fluid.dygraph.to_variable(
+        x)  # TODO(liym27): Delete it if the type of parameter x can be resolved
     flag = 1
     while x < 10:
         i = i + x if flag is not None else x + i
@@ -72,6 +81,10 @@ def for_loop_dyfunc(max_len):
 
 def while_loop_bool_op(x):
     i = fluid.dygraph.to_variable(x)
+
+    # Use `to_variable` so that static analysis can analyze the type of X is Tensor
+    x = fluid.dygraph.to_variable(
+        x)  # TODO(liym27): Delete it if the type of parameter x can be resolved
     while (x >= 0 and x < 10) or x <= -1 or x < -3 or (x < -7 or x < -5):
         i = i + x
         x = x + 1
@@ -102,6 +115,11 @@ def for_loop_class_var(max_len):
             self.c = 5
 
     foo = Foo()
+
+    # Use `to_variable` so that static analysis can analyze the type of X is Tensor
+    # TODO(liym27): Delete it if the type of parameter x can be resolved
+    max_len = fluid.layers.fill_constant(
+        shape=[1], value=max_len, dtype="int32")
     for i in range(max_len):
         foo.b = fluid.layers.zeros(shape=[1], dtype='float32')
         foo.c = foo.b + foo.a
@@ -149,19 +167,17 @@ class TestTransformWhileLoop(unittest.TestCase):
         self.dyfunc = while_loop_dyfunc
 
     def _run_static(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            x_var = fluid.layers.assign(self.x)
-            static_func = dygraph_to_static_func(self.dyfunc)
-
-            out = static_func(x_var)
-            exe = fluid.Executor(self.place)
-            ret = exe.run(main_program, fetch_list=out)
-        return ret
+        return self._run(to_static=True)
 
     def _run_dygraph(self):
+        return self._run(to_static=False)
+
+    def _run(self, to_static):
         with fluid.dygraph.guard(self.place):
-            ret = self.dyfunc(fluid.dygraph.to_variable(self.x))
+            if to_static:
+                ret = declarative(self.dyfunc)(self.x)
+            else:
+                ret = self.dyfunc(self.x)
             return ret.numpy()
 
     def test_ast_to_func(self):
@@ -201,22 +217,20 @@ class TestTransformForLoop(unittest.TestCase):
         self.dyfunc = for_loop_dyfunc
 
     def _run_static(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            static_func = dygraph_to_static_func(self.dyfunc)
-            out = static_func(self.len)
-            exe = fluid.Executor(self.place)
-            ret = exe.run(main_program, fetch_list=out)
-        return ret
+        return self._run(to_static=True)
 
     def _run_dygraph(self):
+        return self._run(to_static=False)
+
+    def _run(self, to_static):
         with fluid.dygraph.guard(self.place):
-            ret = self.dyfunc(self.len)
+            if to_static:
+                ret = declarative(self.dyfunc)(self.len)
+            else:
+                ret = self.dyfunc(self.len)
             return ret.numpy()
 
     def test_ast_to_func(self):
-        static_numpy = self._run_static()
-        self._run_dygraph()
         self.assertTrue(np.allclose(self._run_dygraph(), self._run_static()))
 
 

python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py

@@ -21,9 +21,14 @@ import numpy as np
 
 import paddle
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph.base import switch_to_static_graph
+from paddle.fluid.dygraph import to_variable
 from paddle.fluid.dygraph.nn import Conv2D, Linear, Pool2D
 from paddle.fluid.optimizer import AdamOptimizer
+from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
+
+SEED = 2020
 
 
 class SimpleImgConvPool(fluid.dygraph.Layer):
@@ -94,7 +99,7 @@ class MNIST(fluid.dygraph.Layer):
                     loc=0.0, scale=scale)),
             act="softmax")
 
-    @dygraph_to_static_func
+    @declarative
     def forward(self, inputs, label=None):
         x = self.inference(inputs)
         if label is not None:
@@ -125,62 +130,96 @@ class TestMNIST(unittest.TestCase):
             drop_last=True)
 
 
-class TestMNISTWithStaticMode(TestMNIST):
+class TestMNISTWithDeclarative(TestMNIST):
     """
-    Tests model when using `dygraph_to_static_func` to convert dygraph into static
-    model. It allows user to add customized code to train static model, such as `with`
-    and `Executor` statement.
+    Tests model if doesn't change the layers while decorated
+    by `dygraph_to_static_output`. In this case, everything should
+    still works if model is trained in dygraph mode.
     """
 
-    def test_train(self):
+    def train_static(self):
+        return self.train(to_static=True)
+
+    def train_dygraph(self):
+        return self.train(to_static=False)
+
+    def test_mnist_declarative(self):
+        dygraph_loss = self.train_dygraph()
+        static_loss = self.train_static()
+        self.assertTrue(
+            np.allclose(dygraph_loss, static_loss),
+            msg='dygraph is {}\n static_res is \n{}'.format(dygraph_loss,
+                                                            static_loss))
 
-        main_prog = fluid.Program()
-        with fluid.program_guard(main_prog):
+    def train(self, to_static=False):
+        prog_trans = ProgramTranslator()
+        prog_trans.enable(to_static)
+
+        loss_data = []
+        with fluid.dygraph.guard(self.place):
+            fluid.default_main_program().random_seed = SEED
+            fluid.default_startup_program().random_seed = SEED
             mnist = MNIST()
             adam = AdamOptimizer(
                 learning_rate=0.001, parameter_list=mnist.parameters())
 
-            exe = fluid.Executor(self.place)
-            start = time()
-
-            img = fluid.data(
-                name='img', shape=[None, 1, 28, 28], dtype='float32')
-            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
-            label.stop_gradient = True
-
-            prediction, acc, avg_loss = mnist(img, label)
-            adam.minimize(avg_loss)
-        exe.run(fluid.default_startup_program())
-
-        for epoch in range(self.epoch_num):
-            for batch_id, data in enumerate(self.train_reader()):
-                dy_x_data = np.array([x[0].reshape(1, 28, 28)
-                                      for x in data]).astype('float32')
-                y_data = np.array(
-                    [x[1] for x in data]).astype('int64').reshape(-1, 1)
-
-                out = exe.run(main_prog,
-                              fetch_list=[avg_loss, acc],
-                              feed={'img': dy_x_data,
-                                    'label': y_data})
-                if batch_id % 100 == 0:
-                    print(
-                        "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}"
-                        .format(epoch, batch_id,
-                                np.array(out[0]),
-                                np.array(out[1]), time() - start))
-                    if batch_id == 300:
-                        # The accuracy of mnist should converge over 0.9 after 300 batch.
-                        accuracy = np.array(out[1])
-                        self.assertGreater(
-                            accuracy,
-                            0.9,
-                            msg="The accuracy {} of mnist should converge over 0.9 after 300 batch."
-                            .format(accuracy))
+            for epoch in range(self.epoch_num):
+                start = time()
+                for batch_id, data in enumerate(self.train_reader()):
+                    dy_x_data = np.array(
+                        [x[0].reshape(1, 28, 28)
+                         for x in data]).astype('float32')
+                    y_data = np.array(
+                        [x[1] for x in data]).astype('int64').reshape(-1, 1)
+
+                    img = to_variable(dy_x_data)
+                    label = to_variable(y_data)
+
+                    label.stop_gradient = True
+                    prediction, acc, avg_loss = mnist(img, label=label)
+                    avg_loss.backward()
+
+                    adam.minimize(avg_loss)
+                    loss_data.append(avg_loss.numpy()[0])
+                    # save checkpoint
+                    mnist.clear_gradients()
+                    if batch_id % 10 == 0:
+                        print(
+                            "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}"
+                            .format(epoch, batch_id,
+                                    avg_loss.numpy(),
+                                    acc.numpy(), time() - start))
+                        start = time()
+                    if batch_id == 50:
+                        mnist.eval()
+                        prediction, acc, avg_loss = mnist(img, label)
+                        loss_data.append(avg_loss.numpy()[0])
+                        self.check_save_inference_model([dy_x_data, y_data],
+                                                        prog_trans, to_static,
+                                                        prediction)
                         break
+        return loss_data
+
+    @switch_to_static_graph
+    def check_save_inference_model(self, inputs, prog_trans, to_static, gt_out):
+        if to_static:
+            infer_model_path = "./test_mnist_inference_model"
+            prog_trans.save_inference_model(infer_model_path)
+            infer_out = self.load_and_run_inference(infer_model_path, inputs)
+            self.assertTrue(np.allclose(gt_out.numpy(), infer_out))
+
+    def load_and_run_inference(self, model_path, inputs):
+        exe = fluid.Executor(self.place)
+        [inference_program, feed_target_names,
+         fetch_targets] = fluid.io.load_inference_model(
+             dirname=model_path, executor=exe)
+        assert len(inputs) == len(feed_target_names)
+        results = exe.run(inference_program,
+                          feed=dict(zip(feed_target_names, inputs)),
+                          fetch_list=fetch_targets)
+
+        return np.array(results[0])
 
 
-# TODO: TestCase with cached program is required when building program in `for` loop.
-
 if __name__ == "__main__":
     unittest.main()

python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py

@@ -31,22 +31,24 @@ from ifelse_simple_func import dyfunc_with_if_else
 np.random.seed(0)
 
 
+# TODO(Aurelius): Currently, `declarative` don't support decorate the function
+# that contains layers with initialized operation, like `fc = linear(10, 3)`.
+# Because initialized ops will be added into program and be executed many times.
+# The parameters are assumed to initialized outside of the function.
 def simple_func(x, weight_numpy):
-    weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy)
-    linear = Linear(32, 64, param_attr=weight_initalizer, bias_attr=False)
     x = fluid.dygraph.to_variable(x)
-    y = linear(x)
-    z = linear(x)
+    w = fluid.dygraph.to_variable(weight_numpy)
+    y = fluid.layers.matmul(x, w)
+    z = fluid.layers.mean(y)
     return z
 
 
 @declarative
 def decorated_simple_func(x, weight_numpy):
-    weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy)
-    linear = Linear(32, 64, param_attr=weight_initalizer, bias_attr=False)
     x = fluid.dygraph.to_variable(x)
-    y = linear(x)
-    z = linear(x)
+    w = fluid.dygraph.to_variable(weight_numpy)
+    y = fluid.layers.matmul(x, w)
+    z = fluid.layers.mean(y)
     return z
 
 
@@ -100,6 +102,14 @@ class StaticCode2():
         return x_v
 
 
+class NetWithError(fluid.dygraph.layers.Layer):
+    @declarative
+    def forward(self, x):
+        linear = fluid.dygraph.Linear(32, 64)
+        y = linear(x)
+        return y
+
+
 class TestDygraphToStaticCode(unittest.TestCase):
     def setUp(self):
         # set to print all string diff when assertEqual fails
@@ -120,75 +130,79 @@ class TestDygraphToStaticCode(unittest.TestCase):
 
 
 class TestEnableDeclarative(unittest.TestCase):
-    def test_enable_disable_get_output(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
+    def setUp(self):
+        self.x = np.random.randn(30, 10, 32).astype('float32')
+        self.weight = np.random.randn(32, 64).astype('float32')
+        self.program_translator = ProgramTranslator()
 
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
-            program_translator.enable(True)
-            static_output = program_translator.get_output(simple_func, x,
-                                                          weight)
+    def test_raise_error(self):
+        with fluid.dygraph.guard():
+            self.program_translator.enable(True)
+            net = NetWithError()
+            with self.assertRaises(ValueError):
+                net(fluid.dygraph.to_variable(self.x))
 
-        program_translator.enable(False)
+    def test_enable_disable_get_output(self):
+        self.program_translator.enable(True)
         with fluid.dygraph.guard():
-            dygraph_output = program_translator.get_output(simple_func, x,
-                                                           weight)
+            static_output = self.program_translator.get_output(
+                simple_func, self.x, self.weight)
+
+        self.program_translator.enable(False)
+        with fluid.dygraph.guard():
+            dygraph_output = self.program_translator.get_output(
+                simple_func, self.x, self.weight)
             self.assertTrue(
                 np.allclose(
                     static_output.numpy(), dygraph_output.numpy(), atol=1e-4))
 
     def test_enable_disable_get_func(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
-            program_translator.enable(True)
-            static_func = program_translator.get_func(simple_func)
+        self.program_translator.enable(True)
+        with fluid.dygraph.guard():
+            static_func = self.program_translator.get_func(simple_func)
             self.assertTrue(callable(static_func))
-            static_output = static_func(x, weight)
+            static_output = static_func(self.x, self.weight)
             self.assertTrue(isinstance(static_output, fluid.Variable))
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_func = program_translator.get_func(simple_func)
+            dygraph_func = self.program_translator.get_func(simple_func)
             self.assertTrue(callable(dygraph_func))
-            dygraph_output = dygraph_func(x, weight)
+            dygraph_output = dygraph_func(self.x, self.weight)
             self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase))
 
     def test_enable_disable_get_program(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
-            program_translator.enable(True)
-            static_output = program_translator.get_program(simple_func, x,
-                                                           weight)
-            self.assertTrue(isinstance(static_output, tuple))
-            self.assertEqual(len(static_output), 4)
-            self.assertTrue(isinstance(static_output[0], fluid.Program))
-            self.assertTrue(isinstance(static_output[1], fluid.Program))
-
-        program_translator.enable(False)
+        self.program_translator.enable(True)
+        static_output = self.program_translator.get_program(simple_func, self.x,
+                                                            self.weight)
+        self.assertTrue(isinstance(static_output, tuple))
+        self.assertEqual(len(static_output), 4)
+        self.assertTrue(isinstance(static_output[0], fluid.Program))
+        self.assertTrue(isinstance(static_output[1], fluid.Program))
+        # Check all inputs and outputs are Variable
+        for var in static_output[2]:
+            self.assertTrue(isinstance(var, fluid.Variable))
+
+        for var in static_output[3]:
+            self.assertTrue(isinstance(var, fluid.Variable))
+
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_output = program_translator.get_program(simple_func, x,
-                                                            weight)
+            dygraph_output = self.program_translator.get_program(
+                simple_func, self.x, self.weight)
             self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase))
 
     def test_enable_disable_declarative(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
-            program_translator.enable(True)
-            static_output = decorated_simple_func(x, weight)
+        self.program_translator.enable(True)
+        with fluid.dygraph.guard():
+            static_output = decorated_simple_func(self.x, self.weight)
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_output = decorated_simple_func(x, weight)
+            dygraph_output = decorated_simple_func(self.x, self.weight)
             self.assertTrue(
                 np.allclose(
                     static_output.numpy(), dygraph_output.numpy(), atol=1e-4))

python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py

+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
+import logging
+import time
+import unittest
+
+import numpy as np
+
+import paddle.fluid as fluid
+from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
+from paddle.fluid.dygraph.base import to_variable
+from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.nn import Embedding
+from paddle.fluid.optimizer import SGDOptimizer
+
+PRINT_STEP = 20
+SEED = 2020
+
+program_translator = ProgramTranslator()
+
+
+class SimpleLSTMRNN(fluid.Layer):
+    def __init__(self,
+                 hidden_size,
+                 num_steps,
+                 num_layers=2,
+                 init_scale=0.1,
+                 dropout=None):
+        super(SimpleLSTMRNN, self).__init__()
+        self._hidden_size = hidden_size
+        self._num_layers = num_layers
+        self._init_scale = init_scale
+        self._dropout = dropout
+        self._num_steps = num_steps
+        self.cell_array = []
+        self.hidden_array = []
+
+        self.weight_1_arr = []
+        self.weight_2_arr = []
+        self.bias_arr = []
+        self.mask_array = []
+
+        for i in range(self._num_layers):
+            weight_1 = self.create_parameter(
+                attr=fluid.ParamAttr(
+                    initializer=fluid.initializer.UniformInitializer(
+                        low=-self._init_scale, high=self._init_scale)),
+                shape=[self._hidden_size * 2, self._hidden_size * 4],
+                dtype="float32",
+                default_initializer=fluid.initializer.UniformInitializer(
+                    low=-self._init_scale, high=self._init_scale))
+            self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1))
+            bias_1 = self.create_parameter(
+                attr=fluid.ParamAttr(
+                    initializer=fluid.initializer.UniformInitializer(
+                        low=-self._init_scale, high=self._init_scale)),
+                shape=[self._hidden_size * 4],
+                dtype="float32",
+                default_initializer=fluid.initializer.Constant(0.0))
+            self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1))
+
+    def forward(self, input_embedding, init_hidden=None, init_cell=None):
+        cell_array = []
+        hidden_array = []
+
+        for i in range(self._num_layers):
+            hidden_array.append(init_hidden[i])
+            cell_array.append(init_cell[i])
+
+        res = []
+        for index in range(self._num_steps):
+            step_input = input_embedding[:, index, :]
+            for k in range(self._num_layers):
+                pre_hidden = hidden_array[k]
+                pre_cell = cell_array[k]
+                weight_1 = self.weight_1_arr[k]
+                bias = self.bias_arr[k]
+
+                nn = fluid.layers.concat([step_input, pre_hidden], 1)
+                gate_input = fluid.layers.matmul(x=nn, y=weight_1)
+
+                gate_input = fluid.layers.elementwise_add(gate_input, bias)
+                i, j, f, o = fluid.layers.split(
+                    gate_input, num_or_sections=4, dim=-1)
+                c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid(
+                    i) * fluid.layers.tanh(j)
+                m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o)
+                hidden_array[k] = m
+                cell_array[k] = c
+                step_input = m
+
+                if self._dropout is not None and self._dropout > 0.0:
+                    step_input = fluid.layers.dropout(
+                        step_input,
+                        dropout_prob=self._dropout,
+                        dropout_implementation='upscale_in_train')
+            res.append(step_input)
+        real_res = fluid.layers.concat(res, 1)
+        real_res = fluid.layers.reshape(
+            real_res, [-1, self._num_steps, self._hidden_size])
+        last_hidden = fluid.layers.concat(hidden_array, 1)
+        last_hidden = fluid.layers.reshape(
+            last_hidden, shape=[-1, self._num_layers, self._hidden_size])
+        last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2])
+        last_cell = fluid.layers.concat(cell_array, 1)
+        last_cell = fluid.layers.reshape(
+            last_cell, shape=[-1, self._num_layers, self._hidden_size])
+        last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2])
+        return real_res, last_hidden, last_cell
+
+
+class PtbModel(fluid.Layer):
+    def __init__(self,
+                 hidden_size,
+                 vocab_size,
+                 num_layers=2,
+                 num_steps=20,
+                 init_scale=0.1,
+                 dropout=None):
+        super(PtbModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.vocab_size = vocab_size
+        self.init_scale = init_scale
+        self.num_layers = num_layers
+        self.num_steps = num_steps
+        self.dropout = dropout
+        self.simple_lstm_rnn = SimpleLSTMRNN(
+            hidden_size,
+            num_steps,
+            num_layers=num_layers,
+            init_scale=init_scale,
+            dropout=dropout)
+        self.embedding = Embedding(
+            size=[vocab_size, hidden_size],
+            dtype='float32',
+            is_sparse=False,
+            param_attr=fluid.ParamAttr(
+                name='embedding_para',
+                initializer=fluid.initializer.UniformInitializer(
+                    low=-init_scale, high=init_scale)))
+        self.softmax_weight = self.create_parameter(
+            attr=fluid.ParamAttr(),
+            shape=[self.hidden_size, self.vocab_size],
+            dtype="float32",
+            default_initializer=fluid.initializer.UniformInitializer(
+                low=-self.init_scale, high=self.init_scale))
+        self.softmax_bias = self.create_parameter(
+            attr=fluid.ParamAttr(),
+            shape=[self.vocab_size],
+            dtype="float32",
+            default_initializer=fluid.initializer.UniformInitializer(
+                low=-self.init_scale, high=self.init_scale))
+
+    def build_once(self, input, label, init_hidden, init_cell):
+        pass
+
+    @declarative
+    def forward(self, input, label, init_hidden, init_cell):
+
+        init_h = fluid.layers.reshape(
+            init_hidden, shape=[self.num_layers, -1, self.hidden_size])
+
+        init_c = fluid.layers.reshape(
+            init_cell, shape=[self.num_layers, -1, self.hidden_size])
+
+        x_emb = self.embedding(input)
+
+        x_emb = fluid.layers.reshape(
+            x_emb, shape=[-1, self.num_steps, self.hidden_size])
+        if self.dropout is not None and self.dropout > 0.0:
+            x_emb = fluid.layers.dropout(
+                x_emb,
+                dropout_prob=self.dropout,
+                dropout_implementation='upscale_in_train')
+        rnn_out, last_hidden, last_cell = self.simple_lstm_rnn(x_emb, init_h,
+                                                               init_c)
+
+        projection = fluid.layers.matmul(rnn_out, self.softmax_weight)
+        projection = fluid.layers.elementwise_add(projection, self.softmax_bias)
+
+        loss = fluid.layers.softmax_with_cross_entropy(
+            logits=projection, label=label, soft_label=False)
+        loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps])
+        loss = fluid.layers.reduce_mean(loss, dim=[0])
+        loss = fluid.layers.reduce_sum(loss)
+
+        return loss, last_hidden, last_cell
+
+    def debug_emb(self):
+
+        np.save("emb_grad", self.x_emb.gradient())
+
+
+def train(place):
+
+    num_layers = 1
+    batch_size = 4
+    hidden_size = 10
+    num_steps = 3
+    init_scale = 0.1
+    max_epoch = 1
+    dropout = 0.0
+    vocab_size = 1000
+    batch_num = 200
+
+    with fluid.dygraph.guard(place):
+        fluid.default_startup_program().random_seed = SEED
+        fluid.default_main_program().random_seed = SEED
+        ptb_model = PtbModel(
+            hidden_size=hidden_size,
+            vocab_size=vocab_size,
+            num_layers=num_layers,
+            num_steps=num_steps,
+            init_scale=init_scale,
+            dropout=dropout)
+
+        sgd = SGDOptimizer(
+            learning_rate=1e-3, parameter_list=ptb_model.parameters())
+
+        for epoch_id in range(max_epoch):
+
+            total_loss = 0.0
+            iters = 0.0
+            total_sample = 0
+
+            init_hidden_data = np.zeros(
+                (num_layers, batch_size, hidden_size), dtype='float32')
+            init_cell_data = np.zeros(
+                (num_layers, batch_size, hidden_size), dtype='float32')
+
+            init_hidden = to_variable(init_hidden_data)
+            init_cell = to_variable(init_cell_data)
+            for step_id in range(batch_num):
+                x_data = np.arange(12).reshape(4, 3).astype('int64')
+                y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
+                y_data = y_data.reshape((-1, 1))
+
+                x_data = x_data.reshape((-1, num_steps, 1))
+                y_data = y_data.reshape((-1, num_steps, 1))
+
+                x = to_variable(x_data)
+                y = to_variable(y_data)
+
+                dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden,
+                                                            init_cell)
+                out_loss = dy_loss.numpy()
+
+                dy_loss.backward()
+                sgd.minimize(dy_loss)
+                ptb_model.clear_gradients()
+
+                total_loss += out_loss
+                iters += num_steps
+                total_sample += 1
+                if step_id % PRINT_STEP == 0:
+                    if step_id == 0:
+                        logging.info("epoch %d | step %d, loss %0.3f" % (
+                            epoch_id, step_id, total_loss / total_sample))
+                        avg_batch_time = time.time()
+                    else:
+                        speed = PRINT_STEP / (time.time() - avg_batch_time)
+                        logging.info(
+                            "epoch %d | step %d, loss %0.3f, speed %.3f steps/s"
+                            % (epoch_id, step_id, total_loss / total_sample,
+                               speed))
+                        avg_batch_time = time.time()
+
+            return out_loss, last_hidden.numpy(), last_cell.numpy()
+
+
+def train_dygraph(place):
+    program_translator.enable(False)
+    return train(place)
+
+
+def train_static(place):
+    program_translator.enable(True)
+    return train(place)
+
+
+class TestPtb(unittest.TestCase):
+    def setUp(self):
+        self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \
+            else fluid.CPUPlace()
+
+    def test_check_result(self):
+        loss_1, hidden_1, cell_1 = train_static(self.place)
+        loss_2, hidden_2, cell_2 = train_dygraph(self.place)
+
+        self.assertTrue(
+            np.allclose(loss_1, loss_2),
+            msg="static loss: {} \ndygraph loss: {}".format(loss_1, loss_2))
+        self.assertTrue(
+            np.allclose(hidden_1, hidden_2),
+            msg="static hidden: {} \ndygraph acc1: {}".format(hidden_1,
+                                                              hidden_2))
+        self.assertTrue(
+            np.allclose(cell_1, cell_2),
+            msg="static cell: {} \ndygraph cell: {}".format(cell_1, cell_2))
+
+
+if __name__ == '__main__':
+    unittest.main()

python/paddle/fluid/tests/unittests/dygraph_to_static/test_recursive_call.py

@@ -19,14 +19,17 @@ import unittest
 import numpy as np
 
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph import ProgramTranslator
+from paddle.fluid.dygraph import declarative
+
+program_translator = ProgramTranslator()
 
 SEED = 2020
 np.random.seed(SEED)
 
 
 # Use a decorator to test exception
-@dygraph_to_static_func
+@declarative
 def dyfunc_with_if(x_v):
     if fluid.layers.mean(x_v).numpy()[0] > 5:
         x_v = x_v - 1
@@ -35,7 +38,7 @@ def dyfunc_with_if(x_v):
     return x_v
 
 
-@dygraph_to_static_func
+@declarative
 def nested_func(x_v):
     x_v = fluid.dygraph.to_variable(x_v)
 
@@ -57,17 +60,16 @@ class TestRecursiveCall1(unittest.TestCase):
         self.dyfunc = nested_func
 
     def get_dygraph_output(self):
+        program_translator.enable(False)
         with fluid.dygraph.guard():
             res = self.dyfunc(self.input).numpy()
             return res
 
     def get_static_output(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            static_out = self.dyfunc(self.input)
-        exe = fluid.Executor(self.place)
-        static_res = exe.run(main_program, fetch_list=static_out)
-        return static_res[0]
+        program_translator.enable(True)
+        with fluid.dygraph.guard():
+            res = self.dyfunc(self.input).numpy()
+            return res
 
     def test_transformed_static_result(self):
         static_res = self.get_static_output()
@@ -93,14 +95,14 @@ class MyConvLayer(fluid.dygraph.Layer):
             bias_attr=fluid.ParamAttr(
                 initializer=fluid.initializer.Constant(value=0.5)))
 
-    @dygraph_to_static_func
+    @declarative
     def forward(self, inputs):
         y = dyfunc_with_if(inputs)
         y = lambda_fun(y)
         y = self.dymethod(y)
         return y
 
-    @dygraph_to_static_func
+    @declarative
     def dymethod(self, x_v):
         x_v = fluid.layers.assign(x_v)
         return x_v
@@ -120,7 +122,7 @@ class MyLayer(fluid.dygraph.Layer):
             bias_attr=fluid.ParamAttr(
                 initializer=fluid.initializer.Constant(value=0.5)))
 
-    @dygraph_to_static_func
+    @declarative
     def forward(self, inputs):
         h = self.conv(inputs)
         out = self.fc(h)
@@ -134,7 +136,7 @@ class TestRecursiveCall2(unittest.TestCase):
         self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda(
         ) else fluid.CPUPlace()
 
-    def get_dygraph_output(self):
+    def _run(self):
         with fluid.dygraph.guard():
             self.dygraph_func = self.Layer()
             fluid.default_startup_program.random_seed = SEED
@@ -144,21 +146,13 @@ class TestRecursiveCall2(unittest.TestCase):
 
             return res.numpy()
 
-    def get_static_output(self):
-        startup_program = fluid.Program()
-        startup_program.random_seed = SEED
-        main_program = fluid.Program()
-        main_program.random_seed = SEED
-
-        with fluid.program_guard(main_program, startup_program):
-            self.dygraph_func = self.Layer()
-            data = fluid.layers.assign(self.input)
-            static_out = self.dygraph_func(data)
+    def get_dygraph_output(self):
+        program_translator.enable(False)
+        return self._run()
 
-        exe = fluid.Executor(self.place)
-        exe.run(startup_program)
-        static_res = exe.run(main_program, fetch_list=static_out)
-        return static_res[0]
+    def get_static_output(self):
+        program_translator.enable(True)
+        return self._run()
 
     def test_transformed_static_result(self):
         dygraph_res = self.get_dygraph_output()

python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py

@@ -22,8 +22,11 @@ import paddle.fluid as fluid
 
 from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
 from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from
 
-np.random.seed(2020)
+SEED = 2020
+
+np.random.seed(SEED)
 
 place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace(
 )
@@ -36,7 +39,6 @@ class SimpleFcLayer(fluid.dygraph.Layer):
 
     @declarative
     def forward(self, x):
-        x = fluid.dygraph.to_variable(x)
         y = self._linear(x)
         z = self._linear(y)
         out = fluid.layers.mean(z)
@@ -46,37 +48,90 @@ class SimpleFcLayer(fluid.dygraph.Layer):
 class TestDyToStaticSaveInferenceModel(unittest.TestCase):
     def test_save_inference_model(self):
         fc_size = 20
-
-        x = np.random.random((fc_size, fc_size)).astype('float32')
-        layer = SimpleFcLayer(fc_size)
-
-        program_translator = ProgramTranslator.get_instance()
-        adam = fluid.optimizer.SGD(learning_rate=0.001)
-        program_translator.set_optimizer(adam, index_of_loss=0)
-
-        for i in range(5):
-            out = layer(x)
-
-        main_program = ProgramTranslator.get_instance().main_program
-        expected_persistable_vars = set(
-            [layer._linear.weight.name, layer._linear.bias.name])
+        x_data = np.random.random((fc_size, fc_size)).astype('float32')
+        with fluid.dygraph.guard(place):
+            fluid.default_startup_program().random_seed = SEED
+            fluid.default_main_program().random_seed = SEED
+
+            x = fluid.dygraph.to_variable(x_data)
+            layer = SimpleFcLayer(fc_size)
+            adam = fluid.optimizer.SGD(learning_rate=0.1,
+                                       parameter_list=layer.parameters())
+
+            for i in range(5):
+                loss, _ = layer(x)
+                loss.backward()
+                adam.minimize(loss)
+                layer.clear_gradients()
+            # Check the correctness of the inference
+            dygraph_out, _ = layer(x)
+        self.check_save_inference_model(layer, [x_data], dygraph_out.numpy())
+        self.check_save_inference_model(
+            layer, [x_data], dygraph_out.numpy(), fetch=[0])
+        self.check_save_inference_model(
+            layer, [x_data], dygraph_out.numpy(), feed=[0])
+
+    def check_save_inference_model(self,
+                                   model,
+                                   inputs,
+                                   gt_out,
+                                   feed=None,
+                                   fetch=None):
+        program_translator = ProgramTranslator()
+        expected_persistable_vars = set([p.name for p in model.parameters()])
 
         infer_model_dir = "./test_dy2stat_save_inference_model"
-        ProgramTranslator.get_instance().save_inference_model(infer_model_dir)
-        saved_var_names = set([
-            filename for filename in os.listdir(infer_model_dir)
-            if filename != '__model__'
-        ])
-        self.assertEqual(saved_var_names, expected_persistable_vars)
-
-        infer_model_dir = "./test_dy2stat_save_inference_model_with_fetch"
-        ProgramTranslator.get_instance().save_inference_model(
-            infer_model_dir, fetch=[0])
+        program_translator.save_inference_model(
+            infer_model_dir, feed=feed, fetch=fetch)
         saved_var_names = set([
             filename for filename in os.listdir(infer_model_dir)
             if filename != '__model__'
         ])
         self.assertEqual(saved_var_names, expected_persistable_vars)
+        # Check the correctness of the inference
+        infer_out = self.load_and_run_inference(infer_model_dir, inputs)
+        self.assertTrue(np.allclose(gt_out, infer_out))
+
+    def load_and_run_inference(self, model_path, inputs):
+        exe = fluid.Executor(place)
+        [inference_program, feed_target_names,
+         fetch_targets] = fluid.io.load_inference_model(
+             dirname=model_path, executor=exe)
+        results = exe.run(inference_program,
+                          feed=dict(zip(feed_target_names, inputs)),
+                          fetch_list=fetch_targets)
+
+        return np.array(results[0])
+
+
+class TestPartialProgramRaiseError(unittest.TestCase):
+    def test_param_type(self):
+        program_translator = ProgramTranslator()
+        program_translator.enable(True)
+        x_data = np.random.random((20, 20)).astype('float32')
+
+        with fluid.dygraph.guard(fluid.CPUPlace()):
+            net = SimpleFcLayer(20)
+            x = fluid.dygraph.to_variable(x_data)
+            out = net(x)
+
+            program_cache = program_translator.get_program_cache()
+            _, (concrete_program, _) = program_cache.last()
+
+            params = concrete_program.parameters
+
+            concrete_program.parameters = params[0]
+            # TypeError: Type of self._params should be list or tuple,
+            # but received <class 'paddle.fluid.framework.ParamBase'>.
+            with self.assertRaises(TypeError):
+                partial_program_from(concrete_program)
+
+            params[0] = "linear.w.0"
+            concrete_program.parameters = params
+            # TypeError: Type of self._params[0] should be framework.ParamBase,
+            # but received <type 'str'>.
+            with self.assertRaises(TypeError):
+                partial_program_from(concrete_program)
 
 
 if __name__ == '__main__':

python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py

@@ -18,10 +18,10 @@ import unittest
 
 import numpy as np
 import paddle.fluid as fluid
-import paddle.fluid.framework as framework
 
 from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
-from paddle.fluid.dygraph.nn import Linear
+from paddle.fluid.optimizer import AdamOptimizer
+from test_fetch_feed import Linear
 
 np.random.seed(2020)
 
@@ -29,53 +29,50 @@ place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace(
 )
 
 
-def simple_func(x, weight_numpy):
-    weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy)
-    linear = Linear(32, 64, param_attr=weight_initalizer)
-    x = fluid.dygraph.to_variable(x)
-    y = linear(x)
-    z = linear(x)
-    return z
+class TestDyToStaticSaveLoad(unittest.TestCase):
+    def test_save_load_same_result(self):
+        program_translator = ProgramTranslator()
+        x_data = np.random.randn(30, 10, 32).astype('float32')
+        batch_num = 3
+
+        with fluid.dygraph.guard(place):
 
+            program_translator.enable(True)
+            x = fluid.dygraph.to_variable(x_data)
+            net = Linear(32, 64)
+            adam = AdamOptimizer(
+                learning_rate=0.1, parameter_list=net.parameters())
+
+            for i in range(batch_num):
+                static_out, static_loss = net(x)
+                # Update parameters
+                static_loss.backward()
+                adam.minimize(static_loss)
+                net.clear_gradients()
+            # Save parameters
+            fluid.save_dygraph(net.state_dict(), "./test_dy2stat_save_load")
+            # minimize() will update parameter, call net() to get output and avg_loss.
+            # Switch into eval mode.
+            net.eval()
+            static_out, static_loss = net(x)
+
+        # load parameters into dygraph
+        with fluid.dygraph.guard(place):
+            dygraph_net = Linear(32, 64)
 
-def decorated_simple_func(x, weight_numpy):
-    weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy)
-    linear = Linear(32, 64, param_attr=weight_initalizer)
-    x = fluid.dygraph.to_variable(x)
-    y = linear(x)
-    z = linear(x)
-    return z
+            # Load parameters
+            model_dict, _ = fluid.load_dygraph("./test_dy2stat_save_load")
+            dygraph_net.set_dict(model_dict)
+            # Switch into eval mode.
+            dygraph_net.eval()
 
+            x = fluid.dygraph.to_variable(x_data)
+            # predict output
+            program_translator.enable(False)
+            dygraph_out, dygraph_loss = dygraph_net(x)
 
-class TestDyToStaticSaveLoad(unittest.TestCase):
-    def test_save_load_same_result(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        with fluid.dygraph.guard(place):
-            dygraph_result = simple_func(x, weight)
-
-        main_program, startup_program, inputs, outputs = ProgramTranslator(
-        ).get_program(decorated_simple_func, x, weight)
-        exe = fluid.Executor(place)
-        exe.run(startup_program)
-        fluid.save(main_program, "./test_dy2stat_save_load")
-
-        # set vars to zero so that we can test load in same file
-        for var in main_program.list_vars():
-            if isinstance(var, framework.Parameter) or var.persistable:
-                tensor = fluid.global_scope().find_var(var.name).get_tensor()
-                tensor.set(np.zeros_like(np.array(tensor)), place)
-
-                # make sure all the paramerter or optimizer var have been set to zero
-                tensor_np = np.array(fluid.global_scope().find_var(var.name)
-                                     .get_tensor())
-                self.assertEqual(0, np.sum(np.abs(tensor_np)))
-
-        fluid.load(main_program, "./test_dy2stat_save_load")
-        static_result = exe.run(main_program,
-                                feed={inputs[0].name: x},
-                                fetch_list=outputs)
-        self.assertTrue(np.allclose(dygraph_result.numpy(), static_result))
+        self.assertTrue(np.allclose(dygraph_out.numpy(), static_out.numpy()))
+        self.assertTrue(np.allclose(dygraph_loss.numpy(), static_loss.numpy()))
 
 
 if __name__ == '__main__':

python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py

@@ -22,8 +22,9 @@ import numpy as np
 import paddle
 import paddle.fluid as fluid
 from paddle.fluid.dygraph.base import to_variable
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
 from paddle.fluid.dygraph.nn import BatchNorm, Conv2D, Linear, Pool2D
+from paddle.fluid.dygraph import declarative
+from paddle.fluid.dygraph import ProgramTranslator
 
 SEED = 2020
 np.random.seed(SEED)
@@ -286,7 +287,7 @@ class SeResNeXt(fluid.dygraph.Layer):
             param_attr=fluid.param_attr.ParamAttr(
                 initializer=fluid.initializer.Uniform(-stdv, stdv)))
 
-    @dygraph_to_static_func
+    @declarative
     def forward(self, inputs, label):
         if self.layers == 50 or self.layers == 101:
             y = self.conv0(inputs)
@@ -314,7 +315,10 @@ class SeResNeXt(fluid.dygraph.Layer):
         return out, avg_loss, acc_top1, acc_top5
 
 
-def train_dygraph(train_reader):
+def train(train_reader, to_static):
+    program_translator = ProgramTranslator()
+    program_translator.enable(to_static)
+
     np.random.seed(SEED)
 
     with fluid.dygraph.guard(place):
@@ -374,75 +378,6 @@ def train_dygraph(train_reader):
         )
 
 
-def train_static(train_reader):
-    np.random.seed(SEED)
-    exe = fluid.Executor(place)
-
-    main_prog = fluid.Program()
-    startup_prog = fluid.Program()
-    with fluid.program_guard(main_prog, startup_prog):
-        with fluid.unique_name.guard():
-            main_prog.random_seed = SEED
-            startup_prog.random_seed = SEED
-            img = fluid.data(
-                name="img", shape=[None, 3, 224, 224], dtype="float32")
-            label = fluid.data(name="label", shape=[None, 1], dtype="int64")
-            label.stop_gradient = True
-
-            se_resnext = SeResNeXt()
-            pred, avg_loss_, acc_top1_, acc_top5_ = se_resnext(img, label)
-
-            optimizer = optimizer_setting(train_parameters,
-                                          se_resnext.parameters())
-            optimizer.minimize(avg_loss_)
-
-    exe.run(startup_prog)
-
-    for epoch_id in range(EPOCH_NUM):
-        total_loss = 0.0
-        total_acc1 = 0.0
-        total_acc5 = 0.0
-        total_sample = 0
-        step_idx = 0
-        speed_list = []
-        for step_id, data in enumerate(train_reader()):
-            dy_x_data = np.array([x[0].reshape(3, 224, 224)
-                                  for x in data]).astype('float32')
-            y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-                BATCH_SIZE, 1)
-
-            pred_, avg_loss, acc_top1, acc_top5 = exe.run(
-                main_prog,
-                feed={"img": dy_x_data,
-                      "label": y_data},
-                fetch_list=[pred, avg_loss_, acc_top1_, acc_top5_])
-
-            total_loss += avg_loss
-            total_acc1 += acc_top1
-            total_acc5 += acc_top5
-            total_sample += 1
-
-            if step_id % PRINT_STEP == 0:
-                if step_id == 0:
-                    logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" % \
-                            ( epoch_id, step_id, total_loss / total_sample, \
-                              total_acc1 / total_sample, total_acc5 / total_sample))
-                    avg_batch_time = time.time()
-                else:
-                    speed = PRINT_STEP / (time.time() - avg_batch_time)
-                    speed_list.append(speed)
-                    logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" % \
-                           ( epoch_id, step_id, total_loss / total_sample, \
-                             total_acc1 / total_sample, total_acc5 / total_sample, speed))
-                    avg_batch_time = time.time()
-
-            step_idx += 1
-            if step_idx == STEP_NUM:
-                break
-
-        return pred_, avg_loss, acc_top1, acc_top5
-
-
 class TestSeResnet(unittest.TestCase):
     def setUp(self):
         self.train_reader = paddle.batch(
@@ -452,8 +387,10 @@ class TestSeResnet(unittest.TestCase):
             drop_last=True)
 
     def test_check_result(self):
-        pred_1, loss_1, acc1_1, acc5_1 = train_static(self.train_reader)
-        pred_2, loss_2, acc1_2, acc5_2 = train_dygraph(self.train_reader)
+        pred_1, loss_1, acc1_1, acc5_1 = train(
+            self.train_reader, to_static=False)
+        pred_2, loss_2, acc1_2, acc5_2 = train(
+            self.train_reader, to_static=True)
 
         self.assertTrue(
             np.allclose(pred_1, pred_2),

python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py

@@ -69,6 +69,11 @@ def test_slice_in_while_loop(x, iter_num):
 def test_slice_in_for_loop(x, iter_num):
     x = fluid.dygraph.to_variable(x)
     a = []
+    # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor
+    iter_num = fluid.layers.fill_constant(
+        shape=[1], value=iter_num, dtype="int32"
+    )  # TODO(liym27): Delete it if the type of parameter iter_num can be resolved
+
     for i in range(iter_num):
         a.append(x)
 

python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py

@@ -18,7 +18,7 @@ import numpy
 
 import unittest
 import paddle.fluid as fluid
-from paddle.fluid.dygraph.jit import dygraph_to_static_func
+from paddle.fluid.dygraph.jit import declarative
 
 
 def dyfunc_tensor_shape_1(x):
@@ -171,20 +171,19 @@ class TestTensorShapeBasic(unittest.TestCase):
     def init_test_func(self):
         self.dygraph_func = dyfunc_tensor_shape_1
 
-    def get_dygraph_output(self):
+    def _run(self, to_static):
         with fluid.dygraph.guard():
-            res = self.dygraph_func(self.input).numpy()
+            if to_static:
+                res = declarative(self.dygraph_func)(self.input).numpy()
+            else:
+                res = self.dygraph_func(self.input).numpy()
             return res
 
-    def get_static_output(self):
-        main_program = fluid.Program()
-        with fluid.program_guard(main_program):
-            static_out = dygraph_to_static_func(self.dygraph_func)(self.input)
-
-        exe = fluid.Executor(self.place)
-        static_res = exe.run(main_program, fetch_list=static_out)
+    def get_dygraph_output(self):
+        return self._run(to_static=False)
 
-        return static_res[0]
+    def get_static_output(self):
+        return self._run(to_static=False)
 
     def test_transformed_static_result(self):
         static_res = self.get_static_output()