From a20ce3eeca03456a29fbcd745119034bce611645 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Fri, 15 May 2020 10:14:29 +0800 Subject: [PATCH] [cherry-pick][Dy2stat] training with @declarative decorator and save_inference_model (#24557) * [Dy2Stat] Add test for ptb model. (#24076) * [Dy2Stat] Add test for ptb model. test=develop * Simplify code for gast.If in is_control_flow_to_transform. test=develop * Move IsControlFlowVisitor to file utils. test=develop * Don't use convert_call for build-in func in CallTransformer. test=develop * Optimize api is_control_flow_to_transform. test=develop * Polish the document of IsControlFlowVisitor. test=develop * Use declarative instead of dygraph_to_static_func. test=develop * [dy2static] Add print transformer and unify print format (#24068) * add print transformer & unify print format, test=develop * remove using of dygraph_to_static_func, test=develop * remove python stdout capture, test=develop * fix compatibility problems for PY2, test=develop * fix detail error, test=develop * fix type analysis bug, test=develop * fix print tuple compatible error in PY2, test=develop * replace get_func to declarative, test=develop * fix detail bug, test=develop * fix some detail problems, test=develop * change visit_call in print transformer, test=develop * [dy2static] Support for static graph training with @declarative decorator (#24259) * support to train in static * support to independent decorator * remove in_dygraph_mode condition in ProgramTranslator * fix import param_guard and add train/eval test=develop * Modify into ShareVarsFromScope and rm __all__ in partial_program test=develop * [Dy2Stat] Optimize loop cond (#24049) * Simplify code for gast.If in is_control_flow_to_transform. * Move IsControlFlowVisitor to file utils. * Don't use convert_call for build-in func in CallTransformer. * Optimize api is_control_flow_to_transform. * Polish the document of IsControlFlowVisitor. * revert modification from #24259 * [dy2stat]Support save_inference_model in program_translator (#24353) * support save_inference_model in program_translator test=develop * fix compatibility with OrderedDict.values() in python3 test=develop * synchronized random_seed test=develop * Polish Error Message test=develop * Fix bug with `if Tensor` in is_control_flow (#24433) * fix bug with `if Tensor` in is_control_flow test=develop * remove continue test=develop * Revert "[dy2static] Add print transformer and unify print format (#24068)" This reverts commit 09dd01900c4af919ed62795ffa8af31dff0cb3a3. * Revert "[dy2static] Add print transformer and unify print format (#24068)" This reverts commit 09dd01900c4af919ed62795ffa8af31dff0cb3a3. * fix sample code in sava_inference_model test=develop Co-authored-by: liym27 <33742067+liym27@users.noreply.github.com> Co-authored-by: Chen Weihang --- paddle/fluid/operators/run_program_op.h | 58 +- python/paddle/fluid/data_feeder.py | 3 +- python/paddle/fluid/dygraph/base.py | 20 + .../dygraph_to_static/ast_transformer.py | 12 +- .../break_continue_transformer.py | 3 - .../dygraph_to_static/call_transformer.py | 13 + .../dygraph_to_static/convert_call_func.py | 49 +- .../dygraph_to_static/ifelse_transformer.py | 180 +---- .../dygraph_to_static/list_transformer.py | 9 +- .../dygraph_to_static/loop_transformer.py | 6 +- .../dygraph_to_static/partial_program.py | 222 +++++++ .../dygraph_to_static/print_transformer.py | 88 +++ .../dygraph_to_static/program_translator.py | 627 +++++++----------- .../dygraph_to_static/static_analysis.py | 3 + .../tensor_shape_transformer.py | 11 +- .../fluid/dygraph/dygraph_to_static/utils.py | 236 ++++++- python/paddle/fluid/dygraph/jit.py | 10 +- python/paddle/fluid/dygraph/layers.py | 5 +- python/paddle/fluid/framework.py | 2 +- .../dygraph_to_static/ifelse_simple_func.py | 30 +- .../dygraph_to_static/test_break_continue.py | 12 +- .../dygraph_to_static/test_cache_program.py | 53 +- .../unittests/dygraph_to_static/test_dict.py | 20 +- .../dygraph_to_static/test_fetch_feed.py | 47 +- .../dygraph_to_static/test_full_name_usage.py | 11 +- .../dygraph_to_static/test_ifelse.py | 46 +- .../dygraph_to_static/test_ifelse_basic.py | 55 +- .../unittests/dygraph_to_static/test_list.py | 8 + .../unittests/dygraph_to_static/test_loop.py | 56 +- .../unittests/dygraph_to_static/test_mnist.py | 135 ++-- .../test_program_translator.py | 118 ++-- .../dygraph_to_static/test_ptb_lm.py | 317 +++++++++ .../dygraph_to_static/test_recursive_call.py | 48 +- .../test_save_inference_model.py | 107 ++- .../dygraph_to_static/test_save_load.py | 87 ++- .../dygraph_to_static/test_se_resnet.py | 85 +-- .../unittests/dygraph_to_static/test_slice.py | 5 + .../dygraph_to_static/test_tensor_shape.py | 21 +- 38 files changed, 1710 insertions(+), 1108 deletions(-) create mode 100644 python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py create mode 100644 python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py create mode 100644 python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py diff --git a/paddle/fluid/operators/run_program_op.h b/paddle/fluid/operators/run_program_op.h index 9e099b2e96..7b67f8fad3 100644 --- a/paddle/fluid/operators/run_program_op.h +++ b/paddle/fluid/operators/run_program_op.h @@ -102,74 +102,50 @@ static void CheckOutputVarStatus(const Variable &src_var, } static void VariableShare(const Variable &src_var, Variable *dst_var) { - // The previous check ensures that the variable type can only be LoDTensor - auto *lod_tensor = dst_var->GetMutable(); - lod_tensor->ShareDataWith(src_var.Get()); - lod_tensor->set_lod(src_var.Get().lod()); -} - -static void ShareVarsIntoScope(const std::vector &vars, - const std::vector &var_names, - framework::Scope *scope) { - for (size_t i = 0; i < vars.size(); ++i) { - auto *var = scope->Var(var_names[i]); - CheckInputVarStatus(*vars[i], var_names[i]); - VariableShare(*vars[i], var); - } -} - -static void VariableCopy(const Variable &src_var, - const platform::Place &dst_place, Variable *dst_var) { // The previous check ensures that the variable type can only be LoDTensor or - // SelectedRows + // SelectedRows. if (src_var.IsType()) { auto *lod_tensor = dst_var->GetMutable(); - TensorCopySync(src_var.Get(), dst_place, lod_tensor); + lod_tensor->ShareDataWith(src_var.Get()); lod_tensor->set_lod(src_var.Get().lod()); } else if (src_var.IsType()) { auto *selected_rows = dst_var->GetMutable(); - TensorCopySync(src_var.Get().value(), dst_place, - selected_rows->mutable_value()); + selected_rows->mutable_value()->ShareDataWith( + src_var.Get().value()); selected_rows->set_rows(src_var.Get().rows()); selected_rows->set_height(src_var.Get().height()); } } -static void ShareVarsFromScope(const std::vector &vars, +static void ShareVarsIntoScope(const std::vector &vars, const std::vector &var_names, framework::Scope *scope) { for (size_t i = 0; i < vars.size(); ++i) { - auto *var = scope->FindVar(var_names[i]); - PADDLE_ENFORCE_NOT_NULL( - var, platform::errors::NotFound("The output variable %s is not in " - "RunProgram(Grad)Op(StaticModelRunner)'" - "s internal scope.", - var_names[i])); - CheckOutputVarStatus(*var, *vars[i], var_names[i]); - VariableShare(*var, vars[i]); + auto *var = scope->Var(var_names[i]); + CheckInputVarStatus(*vars[i], var_names[i]); + VariableShare(*vars[i], var); } } -static void CopyVarsFromScope(const std::vector &vars, - const std::vector &var_names, - const platform::Place &dst_place, - framework::Scope *scope) { +static void ShareVarsFromScope(const std::vector &vars, + const std::vector &var_names, + framework::Scope *scope) { for (size_t i = 0; i < vars.size(); ++i) { if (var_names[i] == framework::kEmptyVarName) { VLOG(2) << "find variable name is " << framework::kEmptyVarName << ", skip it!"; continue; } - auto *var = scope->FindVar(var_names[i]); // NOTE: Here skip not found var is dangerous, if a bug is caused here, // the result is grad calculation error, which will be very hidden! + auto *var = scope->FindVar(var_names[i]); PADDLE_ENFORCE_NOT_NULL( var, platform::errors::NotFound("The output variable %s is not in " "RunProgram(Grad)Op(StaticModelRunner)'" "s internal scope.", var_names[i])); CheckOutputVarStatus(*var, *vars[i], var_names[i]); - VariableCopy(*var, dst_place, vars[i]); + VariableShare(*var, vars[i]); } } @@ -306,11 +282,9 @@ class RunProgramGradOpKernel : public framework::OpKernel { end_op_index, /*create_local_scope=*/false, /*create_vars=*/true, /*keep_kids=*/false); - // Step 4. copy outputs - details::CopyVarsFromScope(input_grad_vars, input_grad_var_names, - ctx.GetPlace(), &scope); - details::CopyVarsFromScope(param_grad_vars, param_grad_names, - ctx.GetPlace(), &scope); + // Step 4. get outputs + details::ShareVarsFromScope(input_grad_vars, input_grad_var_names, &scope); + details::ShareVarsFromScope(param_grad_vars, param_grad_names, &scope); } }; diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index 0783bd8213..26c9354511 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -76,7 +76,8 @@ def check_variable_and_dtype(input, expected_dtype, op_name, extra_message=''): - check_type(input, input_name, Variable, op_name, extra_message) + check_type(input, input_name, (Variable, core.VarBase), op_name, + extra_message) check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message) diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index 525563428b..71d5d8b3dc 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -61,6 +61,26 @@ def program_desc_tracing_guard(enable): _functional_dygraph_context_manager = None +@signature_safe_contextmanager +def param_guard(parameters): + # Note: parameters is a reference of self._parameters + if not framework.in_dygraph_mode() and parameters: + origin_parameters = parameters.copy() + for name, var_base in parameters.items(): + if isinstance(var_base, core.VarBase): + new_var = framework.Parameter( + var_base.block, + var_base.shape, + var_base.dtype, + var_base.type, + name=var_base.name) + parameters[name] = new_var + yield + parameters.update(origin_parameters) + else: + yield + + def enabled(): """ This function checks whether the program runs in dynamic graph mode or not. diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py index 9c51972369..62da42c604 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py @@ -14,8 +14,6 @@ from __future__ import print_function -import astor -import copy # gast is a generic AST to represent Python2 and Python3's Abstract Syntax Tree(AST). # It provides a compatibility layer between the AST of various Python versions, # as produced by ast.parse from the standard ast module. @@ -24,8 +22,6 @@ import gast import inspect import textwrap -from paddle.fluid import unique_name - from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfElseTransformer @@ -34,14 +30,9 @@ from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import LoopTransfor from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import TensorShapeTransformer from paddle.fluid.dygraph.dygraph_to_static.call_transformer import CallTransformer -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from paddle.fluid.dygraph.dygraph_to_static.utils import dygraph_class_to_static_api from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name -from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api, is_dygraph_api, is_to_variable -from paddle.fluid.dygraph.dygraph_to_static.utils import to_assign_node, to_static_ast, update_args_of_func __all__ = ['DygraphToStaticAst', 'convert_to_static'] @@ -142,6 +133,9 @@ def convert_to_static(dyfunc): Converts dygraph function into static function. """ # Get AST from dygraph function + # Note: In Python2, it will raise OSError when inspect function + # with decorator directly and dyfunc.__wrapped__ holds the actual function. + dyfunc = getattr(dyfunc, '__wrapped__', dyfunc) raw_code = inspect.getsource(dyfunc) code = textwrap.dedent(raw_code) root = gast.parse(code) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py index b8c59b9976..f83cd6d302 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py @@ -17,9 +17,6 @@ from __future__ import print_function import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import NodeTestTransformer -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import get_constant_variable_node from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py index 9c128a83c4..b872ab723e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py @@ -32,6 +32,15 @@ class CallTransformer(gast.NodeTransformer): self.wrapper_root = wrapper_root self.root = wrapper_root.node + def _is_builtin_call(self, node): + assert isinstance(node, gast.Call) + func_str = ast_to_source_code(node.func).strip() + try: + from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import is_builtin + return eval("is_builtin({})".format(func_str)) + except Exception: + return False + def transform(self): self.visit(self.root) @@ -39,6 +48,10 @@ class CallTransformer(gast.NodeTransformer): self.generic_visit(node) if is_paddle_api(node): return node + + if self._is_builtin_call(node): + return node + func_str = ast_to_source_code(node.func).strip() new_func_str = "fluid.dygraph.dygraph_to_static.convert_call({})".format( func_str) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py index 775a95cae7..5aa0ffb3e4 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py @@ -103,17 +103,8 @@ def convert_call(func): return func try: if func in func.__globals__.values(): - if six.PY3: - source_code = inspect.getsource(func) - if any(decorator in source_code - for decorator in DECORATOR_NAMES): - converted_call = None - else: - converted_call = to_static_func(func) - func_self = getattr(func, '__self__', None) - else: - converted_call = to_static_func(func) - func_self = getattr(func, '__self__', None) + converted_call = to_static_func(func) + func_self = getattr(func, '__self__', None) except AttributeError: # NOTE: # If func is not in __globals__, it does not need to be transformed @@ -121,45 +112,25 @@ def convert_call(func): converted_call = None except (IOError, OSError): # NOTE: - # If func has beed decorated, its source code can not be get + # If func has been decorated, its source code can not be get # so that it can not be transformed to static function. converted_call = None elif inspect.ismethod(func): try: - if six.PY3: - source_code = inspect.getsource(func) - if any(decorator in source_code - for decorator in DECORATOR_NAMES): - converted_call = None - else: - converted_call = to_static_func(func) - func_self = getattr(func, '__self__', None) - else: - converted_call = to_static_func(func) - func_self = getattr(func, '__self__', None) + converted_call = to_static_func(func) + func_self = getattr(func, '__self__', None) except (IOError, OSError): - # NOTE: func may have beed decorated. + # NOTE: func may have been decorated. converted_call = None elif hasattr(func, '__class__') and hasattr(func.__class__, '__call__'): if hasattr(func, 'forward') and isinstance(func, Layer): try: - if six.PY3: - source_code = inspect.getsource(func.forward) - if any(decorator in source_code - for decorator in DECORATOR_NAMES): - converted_call = None - else: - forward_func = to_static_func(func.forward) - setattr(func, 'forward', forward_func) - func_self = func - else: - forward_func = to_static_func(func.forward) - setattr(func, 'forward', forward_func) - func_self = func - + forward_func = to_static_func(func.forward) + setattr(func, 'forward', forward_func) + func_self = func except Exception: - # NOTE: func.forward may have beed decorated. + # NOTE: func.forward may have been decorated. func_self = None if func_self else func_self converted_call = func else: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py index b57d74f647..d9458007b6 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py @@ -22,15 +22,18 @@ from collections import defaultdict # as produced by ast.parse from the standard ast module. # See details in https://github.com/serge-sans-paille/gast/ import gast -import six from paddle.fluid import unique_name +from paddle.fluid.dygraph.dygraph_to_static.utils import compare_with_none +from paddle.fluid.dygraph.dygraph_to_static.utils import is_candidate_node from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node from paddle.fluid.dygraph.dygraph_to_static.utils import create_assign_node +from paddle.fluid.dygraph.dygraph_to_static.utils import IsControlFlowVisitor from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType TRUE_FUNC_PREFIX = 'true_fn' FALSE_FUNC_PREFIX = 'false_fn' @@ -142,155 +145,25 @@ class IfElseTransformer(gast.NodeTransformer): return self.new_func_nodes -def is_candidate_node(node): - """ - Nodes with specified type will be dependent on tensor. - """ - is_compare_node = isinstance(node, - (gast.Compare, gast.BoolOp, gast.UnaryOp)) - # TODO(Aurelius84): `.numpy()` may be an customized function, - # and should consider a more elegant way to solve this problem. - has_numpy_attr = ".numpy()" in ast_to_source_code(node) - return is_compare_node or has_numpy_attr - - -def compare_with_none(node): - """ - Whether the comparator of `gast.Compare` node is `None`. - """ - if isinstance(node, gast.Compare): - for child in [node.left, node.comparators]: - # node.comparators is a list. - if isinstance(child, list): - child = child[0] - if (isinstance(child, gast.Constant) and child.value is None) or ( - isinstance(child, gast.Name) and child.id == 'None'): - return True - return False - - -class IsControlFlowVisitor(gast.NodeVisitor): - """ - Judge whether the node.test from Dygraph code dependent on paddle Tensor. - If does, it should satisfy: - 1. must involve at least one var whose type is Tensor. - 2. the Tensor var should call `.numpy()[]` interface or Tensor.shape is [1]. - 3. involve Tensor.shape[i] and the shape[i] is unknown in compile time. - The following examples should not be considered as control_flow_if: - 1. `if Tensor_var` or `if Tensor_var is None` - 2. if Tensor.shape[i] is determined with fixed value (not -1 or None) - - Note: pred in ConditionalBlock require variable, which means all vars should be Tensor - or transformed into Tensor, like fill_constant(shape=[1], dtype='int32', value=Tensor.shape[i]). - - TODO: 1. need to deal with `tensor.shape[i]` which need to eval the data of shape[i], - because reshape_op may be called before this statement. - """ - +class NodeTestTransformer(gast.NodeTransformer): def __init__(self, ast_node, - static_analysis_visitor=None, - node_var_type_map=None): - assert isinstance( - ast_node, gast.AST - ), "Type of input node should be gast.AST, but received %s." % type( - ast_node) - self.ast_root = ast_node - if static_analysis_visitor is None: - static_analysis_visitor = StaticAnalysisVisitor(ast_node) - self.static_analysis_visitor = static_analysis_visitor - self.node_var_type_map = node_var_type_map - - self.is_control_flow_num = 0 - self._compare_node_tenor_set = set() - - def transform(self): - node = self.ast_root - if is_candidate_node(node): - self.visit(node) - return self.is_control_flow_num > 0 - - def visit_BoolOp(self, node): - for i, child in enumerate(node.values): - if is_candidate_node(child): - self.visit(child) - return node - - def visit_Compare(self, node): - # Ignores child node with `if x` or `if x is None` - # TODO(Aurelius84): `if tensor` will be supported in dygraph - # and should be considered as is_control_flow. - pre_control_flow_num = self.is_control_flow_num - if not compare_with_none(node): - self.generic_visit(node) - for child in gast.walk(node): - if isinstance(child, gast.Subscript): - self._visit_Subscript(child) - if self.is_control_flow_num > pre_control_flow_num: - self._compare_node_tenor_set.add(node) - return node - - def _visit_Subscript(self, node): - self.generic_visit(node) - if hasattr(node, 'value') and isinstance(node.value, gast.Call): - self._visit_Call(node.value) - return node - - def _visit_Call(self, node): - assert isinstance(node, gast.Call) - if isinstance(node.func, gast.Attribute): - attr_node = node.func - if attr_node.attr == 'numpy': - self.is_control_flow_num += 1 - - def visit_Call(self, node): - self._visit_Call(node) - if is_paddle_api(node): - self.is_control_flow_num += 1 - return node - - def visit_Name(self, node): - if self._is_node_with_tensor(node, node.id): - self.is_control_flow_num += 1 - return node - - def visit_Constant(self, node): - if self._is_node_with_tensor(node, node.value): - self.is_control_flow_num += 1 - return node - - def _is_node_with_tensor(self, node, name_id): - tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES} - # Look up the node_var_type_map by name_id. - if self.node_var_type_map: - if name_id and isinstance(name_id, six.string_types): - var_type = self.node_var_type_map.get(name_id, None) - if var_type and var_type & tensor_types: - return True - # if not found, look up the node_to_wrapper_map by node. - node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( - ) - wrapper_node = node_to_wrapper_map.get(node, None) - if wrapper_node is not None: - if wrapper_node.node_var_type & tensor_types: - return True - - return False - - def get_compare_nodes_with_tensor(self): - return self._compare_node_tenor_set - - -class NodeTestTransformer(gast.NodeTransformer): - def __init__(self, ast_node, compare_nodes_with_tensor=None): + compare_nodes_with_tensor=None, + node_to_wrapper_map=None): if compare_nodes_with_tensor is None: compare_nodes_with_tensor = set() self.ast_root = ast_node self._compare_nodes_with_tensor = compare_nodes_with_tensor + if node_to_wrapper_map is None: + node_to_wrapper_map = {} + self.node_to_wrapper_map = node_to_wrapper_map self._new_assign_nodes = [] def transform(self): - return self.visit(self.ast_root) + node = self.ast_root + if not is_candidate_node(node): + return self._create_cast_node(node) + return self.visit(node) def visit_Call(self, node): # Remove `numpy()` statement, like `Tensor.numpy()[i]` -> `Tensor[i]` @@ -319,8 +192,11 @@ class NodeTestTransformer(gast.NodeTransformer): def visit_BoolOp(self, node): for i, child in enumerate(node.values): if not is_candidate_node(child): - node.values[i] = self._create_bool_node(child) - continue + node_wrapper = self.node_to_wrapper_map.get(child, None) + if node_wrapper and node_wrapper.node_var_type & NodeVarType.TENSOR_TYPES: + node.values[i] = self._create_cast_node(child) + else: + node.values[i] = self._create_bool_node(child) self.generic_visit(node) new_node = self._create_logic_node(node) return new_node @@ -332,10 +208,19 @@ class NodeTestTransformer(gast.NodeTransformer): self.generic_visit(node) return node + def _create_cast_node(self, node): + template = "fluid.layers.cast(x={}, dtype='bool')" + + return self._create_node_with_api_template(node, template) + def _create_bool_node(self, node): + template = "fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool({}))" + + return self._create_node_with_api_template(node, template) + + def _create_node_with_api_template(self, node, template): node_code = ast_to_source_code(node) - new_node_str = "fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool({}))".format( - node_code) + new_node_str = template.format(node_code) # gast.parse return Module(body=[expr(value=...)]) new_node = gast.parse(new_node_str).body[0].value bool_tensor_name = unique_name.generate(PLAIN_TENSOR_PREFIX) @@ -395,7 +280,8 @@ class IfConditionVisitor(object): self.static_analysis_visitor = static_analysis_visitor self.visitor = IsControlFlowVisitor(node, static_analysis_visitor, node_var_type_map) - self.transformer = NodeTestTransformer(node) + self.transformer = NodeTestTransformer( + node, node_to_wrapper_map=self.visitor.node_to_wrapper_map) self.compare_nodes_with_tensor = set() self._is_control_flow_if = False diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py index 5579229ccd..4780325bae 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py @@ -55,19 +55,22 @@ class ListTransformer(gast.NodeTransformer): def visit_If(self, node): self.generic_visit(node) - if is_control_flow_to_transform(node, self.scope_var_type_dict): + if is_control_flow_to_transform(node, self.static_analysis_visitor, + self.scope_var_type_dict): self._transform_list_append_in_control_flow(node) return node def visit_While(self, node): self.generic_visit(node) - if is_control_flow_to_transform(node, self.scope_var_type_dict): + if is_control_flow_to_transform(node, self.static_analysis_visitor, + self.scope_var_type_dict): self._transform_list_append_in_control_flow(node) return node def visit_For(self, node): self.generic_visit(node) - if is_control_flow_to_transform(node, self.scope_var_type_dict): + if is_control_flow_to_transform(node, self.static_analysis_visitor, + self.scope_var_type_dict): self._transform_list_append_in_control_flow(node) return node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py index 82400ab01b..900e48269d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py @@ -26,6 +26,7 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import generate_name_node from paddle.fluid.dygraph.dygraph_to_static.utils import get_constant_variable_node from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name +from paddle.fluid.dygraph.dygraph_to_static.utils import is_control_flow_to_transform from paddle.fluid.dygraph.dygraph_to_static.utils import RenameTransformer from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_static_variable_gast_node from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable_gast_node @@ -150,8 +151,9 @@ class NameVisitor(gast.NodeVisitor): self.visit(root_node) def is_control_flow_loop(self, node): - # TODO: make a better condition - return True + need_transform = is_control_flow_to_transform( + node, self.static_analysis_visitor) + return need_transform def get_loop_var_names(self, node): assert isinstance( diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py new file mode 100644 index 0000000000..31a37e7e2b --- /dev/null +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -0,0 +1,222 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import numpy as np +from paddle.fluid import framework, backward, core +from paddle.fluid.dygraph import layers +from paddle.fluid.dygraph.base import switch_to_static_graph +import paddle.compat as cpt + + +class PartialProgramLayer(layers.Layer): + """ + PartialProgramLayer wraps all the ops from layers decorated by `@declarative` + and execute them as a static subgraph. + + .. note:: + **1. This is a very low level API. Users should not use this API + directly. Please use `partial_program_from(concrete_program)` + to create it. + **2. LoDTensorArray is not currently supported in the output. + + Args: + main_program(Program): The main program that contains ops need to be executed. + inputs(list[Variable]): The input list of the decorated function by `@declarative`. + outputs(list[Variable]): The output list of the decorated function by `@declarative`. + parameters(list[VarBase]|None): All trainable parameters included in the program. Default None. + + Returns: + Layer: A Layer object that run all ops internally in static mode. + """ + + def __init__(self, main_program, inputs, outputs, parameters=None): + super(PartialProgramLayer, self).__init__() + self.inputs = inputs + self.outputs = outputs + self._params = parameters if parameters is not None else [] + # Check all params from main program can be found in self._params: + # 1. parameter in self._params should be type `framework.ParamBase` which are created in dygraph. + # 2. parameter from transformed program shall be found in self._params. + # Because they share same data with ParamBase of original dygraph. + self._check_params_all_inited(main_program) + + self._infer_program = main_program + self._train_program = self._append_backward_desc() + # Switch infer or train by train() and eval() + self._trace_program = None + self._set_grad_type(self._params) + self._inner_scope = core.Scope() + # Set default mode to train + self.train() + + @switch_to_static_graph + def _append_backward_desc(self): + program = self._infer_program.clone() + targets = [] + for out in self.outputs: + if isinstance(out, framework.Variable): + targets.append(program.global_block().var(out.name)) + + if targets and self._params: + backward.gradients(targets=targets, inputs=[]) + + return program + + def train(self): + # self.training is inherited from layers.Layer + self.training = True + self._trace_program = self._train_program + + def eval(self): + self.training = False + self._trace_program = self._infer_program + + def forward(self, inputs): + in_vars, out_vars, tmp_scope_vec = self._prepare(inputs) + + framework._dygraph_tracer().trace_op( + type='run_program', + inputs={ + 'X': valid_vars(in_vars), + 'Params': valid_vars(self._params) + }, + outputs={'Out': valid_vars(out_vars), + 'OutScope': tmp_scope_vec}, + attrs={ + 'global_block': self._trace_program.desc.block(0), + 'start_op_index': 0, + 'end_op_index': self._infer_program.desc.block(0).op_size(), + 'is_test': not self.training + }) + + outs = out_vars + if len(outs) == 1: + outs = outs[0] + return outs + + def _prepare(self, inputs): + """ + Prepare inputs, outputs, attrs. + """ + assert isinstance(inputs, (tuple, list)) + # Convert variable into VarBase and feed in training data. + input_vars = [] + for i, value in enumerate(inputs): + if isinstance(value, np.ndarray): + var = core.VarBase( + value=value, + name=self.inputs[i].desc.name(), + persistable=False, + place=framework._current_expected_place(), + zero_copy=True) + elif isinstance(value, core.VarBase): + var = value + var.name = self.inputs[i].desc.name() + else: + continue + input_vars.append(var) + # Create VarBase to receive output data. + out_vars = [] + for var in self.outputs: + if not isinstance(var, framework.Variable): + continue + var_desc = var.desc + var_base = core.VarBase(var_desc.dtype(), + var_desc.shape(), + var_desc.name(), var_desc.type(), False) + out_vars.append(var_base) + + # Hold forward variables + tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], + "program_out_scope", + core.VarDesc.VarType.STEP_SCOPES, True) + + tmp_scope_vec.value().set_scope(self._inner_scope) + + return input_vars, out_vars, tmp_scope_vec + + def _set_grad_type(self, params): + # NOTE: if user set sparse gradient mode, the param's gradient + # will be SelectedRows, not LoDTensor. But tracer will just + # set param grad VarBase by forward VarBase(LoDTensor) + # If we don't change grad_var type here, RunProgramOp need + # transform SelectedRows to LoDTensor forcibly, it may not + # be user wanted result. + for param in params: + grad_name = param.name + core.grad_var_suffix() + grad_var = self._train_program.desc.block(0).find_var( + cpt.to_bytes(grad_name)) + # NOTE: cannot find var desc maybe no problem, such as in batch_norm + if grad_var is None: + continue + param._set_grad_type(grad_var.type()) + + def _check_params_all_inited(self, main_program): + """ + Check all params from main program are already initialized, see details as follows: + 1. all parameters in self._params should be type `framework.ParamBase` which are created in dygraph. + 2. all parameters from transformed program can be found in self._params. + Because they share same data with ParamBase of original dygraph. + """ + if not isinstance(self._params, (list, tuple)): + raise TypeError( + "Type of self._params in PartialProgramLayer should be list or tuple, but received %s." + % type(self._params)) + + params_name_set = set() + for i, param in enumerate(self._params): + if not isinstance(param, framework.ParamBase): + raise TypeError( + 'Type of self._params[{}] in PartialProgramLayer should be framework.ParamBase, but received {}.'. + format(i, type(param))) + params_name_set.add(param.name) + + for block in main_program.blocks: + for name, var in block.vars.items(): + if isinstance(var, framework.Parameter): + if name not in params_name_set: + raise ValueError( + "\n\tWe don't support to define layer with parameters in the function " + "decorated by `@declarative`.\n\tBecause that will re-defined parameters " + "every time when you run the function.\n\t" + "But we found parameter(%s) was created in the decorated function.\n\t" + "Please define the layer with parameters in `__init__` function." + % name) + + +def valid_vars(vars): + """ + Note: run_program_op.InferShape requires `X`/'Out' not be null. + But it's common in dy2static, fake varBase is created to handle the + problem. + """ + if vars: + return vars + return [ + core.VarBase( + value=[1], + name='Fake_var', + place=framework._current_expected_place()) + ] + + +def partial_program_from(concrete_program): + inputs = concrete_program.inputs + if inputs and isinstance(inputs[0], layers.Layer): + inputs = inputs[1:] + + return PartialProgramLayer(concrete_program.main_program, inputs, + concrete_program.outputs, + concrete_program.parameters) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py new file mode 100644 index 0000000000..6903513333 --- /dev/null +++ b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py @@ -0,0 +1,88 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import gast + +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType, StaticAnalysisVisitor + + +class PrintTransformer(gast.NodeTransformer): + """ + This class transforms python print function to fluid.layers.Print. + """ + + def __init__(self, wrapper_root): + assert isinstance( + wrapper_root, AstNodeWrapper + ), "Input non-AstNodeWrapper node for the initialization of PrintTransformer." + self.wrapper_root = wrapper_root + self.root = wrapper_root.node + + self.static_analysis_visitor = StaticAnalysisVisitor(self.root) + self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + ) + + def transform(self): + self.visit(self.root) + + # NOTE: deal with print in PY3 + def visit_Call(self, node): + assert isinstance(node, gast.Call) + if isinstance(node.func, gast.Name) and node.func.id == 'print': + var = self._get_print_var(node) + return self._construct_print_node(var) + return node + + # NOTE: deal with print in PY2 + def visit_Print(self, node): + var = self._get_print_var(node) + print_call_node = self._construct_print_node(var) + return gast.Expr(value=print_call_node) + + def _get_print_var(self, node): + if isinstance(node, gast.Call): + var_list = node.args + elif isinstance(node, gast.Print): + var_list = node.values + if isinstance(var_list[0], gast.Tuple): + var_list = var_list[0].elts + # TODO: support print multiple Var + assert len(var_list) == 1, "Now only support print one Variable." + return var_list[0] + + def _construct_print_node(self, node): + if isinstance(node, gast.Name): + if self._is_tensor_node(node): + print_node = gast.Call( + func=gast.parse('fluid.layers.Print').body[0].value, + args=[node], + keywords=[]) + return print_node + else: + raise TypeError( + "print object type error, only support print Variable now.") + else: + # TODO: may not only print with format + raise NotImplementedError( + "cannot transform print with format temporarily.") + + def _is_tensor_node(self, node): + tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES} + wrapper_node = self.node_to_wrapper_map.get(node, None) + if wrapper_node is not None: + if wrapper_node.node_var_type & tensor_types: + return True + return False diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index 4d0abd5269..908250666e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -16,19 +16,22 @@ from __future__ import print_function import gast import inspect import logging -import numpy import textwrap import threading - +import collections +import numpy as np +from paddle.fluid import core, scope_guard from paddle.fluid import framework -from paddle.fluid import core, executor -from paddle.fluid.dygraph import guard, to_variable +from paddle.fluid import executor +from paddle.fluid import unique_name +from paddle.fluid.dygraph import layers +from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph.dygraph_to_static.ast_transformer import convert_to_static from paddle.fluid.dygraph.dygraph_to_static.ast_transformer import DygraphToStaticAst from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import data_layer_not_check -from paddle.fluid.framework import in_dygraph_mode +from paddle.fluid.dygraph.base import param_guard from paddle.fluid.data_feeder import check_type +from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from __all__ = ['ProgramTranslator', 'convert_function_with_cache'] @@ -45,14 +48,14 @@ class FunctionCache(object): self._static_func_to_transformer = dict() def get_or_cache_func(self, func): - code = self._get_dedent_code_string(func) - static_func = self._dycode_to_static_func.get(code, None) + # code = self._get_dedent_code_string(func) + static_func = self._dycode_to_static_func.get(func, None) if static_func is None: static_func, dygraph_to_static_transformer = convert_to_static(func) - self._dycode_to_static_func[code] = static_func + self._dycode_to_static_func[func] = static_func self._static_func_to_transformer[ - static_func] = dygraph_to_static_transformer + func] = dygraph_to_static_transformer return static_func @@ -65,8 +68,7 @@ class FunctionCache(object): return dedent_code def exist(self, func): - return self._dycode_to_static_func.get( - self._get_dedent_code_string(func), None) is not None + return self._dycode_to_static_func.get(func, None) is not None _CACHE_LOCK = threading.Lock() @@ -82,166 +84,166 @@ def convert_function_with_cache(dygraph_func): return static_func -def synchronized(func): - func.__lock__ = threading.Lock() +class FunctionSpec(object): + def __init__(self, func, args, kwargs): + self._dyfunc = func + self._args = args + self._kwargs = kwargs - def lock_func(*args, **kwargs): - with func.__lock__: - return func(*args, **kwargs) + def is_method(self): + return self._args and isinstance(self._args[0], layers.Layer) - return lock_func + def parameters(self, include_sublayer=True): + params = collections.OrderedDict() + if self.is_method(): + if include_sublayer: + params = self._args[0].parameters() + names = [p.name for p in params] + params = collections.OrderedDict(zip(names, params)) + else: + params = self._args[0]._parameters + return params + + @switch_to_static_graph + def to_static_inputs(self, main_program): + inputs = [] + block = main_program.global_block() + for input_var in self.args: + if isinstance(input_var, np.ndarray): + feed_layer = block.create_var( + name=unique_name.generate('feed'), + shape=list(input_var.shape), + dtype=input_var.dtype, + is_data=True, + need_check_feed=False) + elif isinstance(input_var, core.VarBase): + feed_layer = block.create_var( + name=input_var.name, + shape=list(input_var.shape), + dtype=input_var.dtype, + stop_gradient=input_var.stop_gradient, + need_check_feed=False) + else: + feed_layer = input_var + inputs.append(feed_layer) + return inputs -class ProgramCache(object): - """ - Wrapper class for the program functions defined by dygraph function. - """ + @property + def dyfunc(self): + return self._dyfunc - def __init__(self): - self._inputs = [] - self._outputs = [] - # Always set program to default_main_program. Because once `__call__` is called, - # it means layers(or Ops) are added into default_main_program switched by outer - # `with` statement. - self._main_program = framework.default_main_program() - self._startup_program = framework.default_startup_program() - self._func_cache = FunctionCache() - self._feed_name_to_idx = {} - # Stores the entry function of Net or Model. - self._forward_func = None - self._is_repeated = False - # Indicates whether the function call is still building program. - # Because user can call recursively when `Net` has sub class in - # `forward()`. - self._in_build_process = True - - def build_program_and_return_output(self, dyfunc, *args, **kwargs): + @property + def args(self): + return self._args + + def __key(self): + # Note: if dygraph function is a method of class, + # consider instance info as hash key. + if self.is_method(): + return self._dyfunc, self._args[0] + else: + return self._dyfunc + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + return self.__key() == self.__key() + + +class ConcreteProgram(object): + def __init__(self, + inputs, + outputs, + parameters, + func, + main_program, + startup_program=None): + self.inputs = inputs + self.outputs = outputs + self.main_program = main_program + self.startup_program = startup_program + self.parameters = parameters + self.func_spec = func + + @staticmethod + @switch_to_static_graph + def from_func_spec(func_spec): """ Builds the main_program with specialized inputs and returns outputs of program as fetch_list. """ # Transforms dygraph function into static function and caches it. - static_func = self._transform_or_cache_layers(dyfunc) + dygaph_function = func_spec.dyfunc + static_func = convert_function_with_cache(dygaph_function) - # 1. Adds `fluid.data` layers for input if needed - if not self._inputs: - self._add_feed_layers(args, kwargs) + main_program, startup_program = framework.Program(), framework.Program() + # Note: The random seed should be synchronized into cached program + # if set in `fluid.dygrap_guard` because some ops rely on it, such as + # `fluid.layers.dropout`. + main_program.random_seed = framework.default_main_program().random_seed + startup_program.random_seed = framework.default_startup_program( + ).random_seed - # 2. Avoids inserting forward ops repeatedly. - if self._is_repeated: - return self.outputs - - # 3. Builds program only once and returns the output Variables. - outputs = self._get_or_build_program(static_func, args, kwargs) - - if static_func == self._forward_func: - self._in_build_process = False + with framework.program_guard(main_program, startup_program): + # 1. Adds `fluid.data` layers for input if needed + inputs = func_spec.to_static_inputs(main_program) - return outputs + # 2. Gets all ParamBases in the function + all_parameters = list(func_spec.parameters().values()) - def _transform_or_cache_layers(self, dyfunc): - """ - Transforms dygraph function into static function. - """ - static_func = self._func_cache.get_or_cache_func(dyfunc) + # 3. Builds program only once and returns the output Variables. + with param_guard(func_spec.parameters(False)): + outputs = static_func(*inputs) + if not isinstance(outputs, (tuple, list)): + outputs = [outputs] if outputs else [] - if self._forward_func is None: - self._forward_func = static_func - else: - # self._forward_func is entry function of Net or Model. - # It can be called for multiple times, but layers from these functions - # call stack will be added into self._main_program only once. - # After that, cached program will be always returned by default. - if static_func == self._forward_func: - self._is_repeated = True - # If a independent function is received after the build process - # has finished, feed layers should be reset. - # TODO(Aurelius84): Switch main_program without specifying program_guard. - elif not self._in_build_process: - self._inputs = [] - self._is_repeated = False - self._forward_func = static_func + return ConcreteProgram( + inputs=inputs, + outputs=outputs, + parameters=all_parameters, + func=dygaph_function, + main_program=main_program, + startup_program=startup_program) - return static_func - def _get_or_build_program(self, func, args, kwargs): - """ - Returns program of the input function. If called at first time, - builds a new program and caches it. - """ - with framework.program_guard(self._main_program, self._startup_program): - if func == self._forward_func: - # Replaces input data with `layers.data` - args = list(args) - for feed_layer in self._inputs: - idx = self.feed_name_to_idx[feed_layer.name] - args[idx] = feed_layer - fetch_list = func(*args, **kwargs) - if not isinstance(fetch_list, tuple): - # func just returns one reuslt - fetch_list = [fetch_list] - fetch_list = list(fetch_list) - self._outputs = fetch_list - else: - fetch_list = func(*args, **kwargs) - if not isinstance(fetch_list, tuple): - # func just returns one reuslt - fetch_list = [fetch_list] - fetch_list = list(fetch_list) +class ProgramCache(object): + """ + Wrapper class for the program functions defined by dygraph function. + """ - return fetch_list + def __init__(self): + self._caches = collections.OrderedDict() - def _add_feed_layers(self, args, kwargs): - """ - Adds `fluid.data` if the input `numpy.ndarray` is converted into `Variable` - by `to_variable()`, it makes program to be executed dynamically. - """ - self._feed_name_to_idx = self._get_name_to_idx(self._forward_func) - with framework.program_guard(self._main_program, self._startup_program): - for feed_name, idx in self.feed_name_to_idx.items(): - batch_data = args[idx] - assert isinstance( - batch_data, numpy.ndarray - ), "Input {} should be numpy.ndarray, but received {}.".format( - feed_name, type(batch_data)) - feed_layer = data_layer_not_check( - name=feed_name, - shape=list(batch_data.shape), - dtype=str(batch_data.dtype)) - self._inputs.append(feed_layer) - - def _get_name_to_idx(self, func): - """ - Returns name and index of input args from `forward(args)` - that need to be replaced with `fluid.data`. - """ - transformer = self._func_cache.get_transformer(func) - feed_name_to_idx = transformer.get_feed_name_to_idx() - return feed_name_to_idx + def _build_once(self, func_spec): + concrete_program = ConcreteProgram.from_func_spec(func_spec) + return concrete_program, partial_program_from(concrete_program) - @property - def main_program(self): - return self._main_program + def __getitem__(self, item): + if not isinstance(item, FunctionSpec): + raise ValueError( + 'type(item) should be FunctionSpec, but received %s' % + type(item)) + if item not in self._caches: + self._caches[item] = self._build_once(item) + return self._caches[item] - @property - def startup_program(self): - return self._startup_program + def last(self): + assert len( + self._caches) >= 1, "No valid cached program in ProgramCache." + key = next(reversed(self._caches.keys())) + return key, self._caches[key] - @property - def inputs(self): - return self._inputs - @property - def outputs(self): - return self._outputs +def synchronized(func): + func.__lock__ = threading.Lock() - @property - def feed_name_to_idx(self): - return self._feed_name_to_idx + def lock_func(*args, **kwargs): + with func.__lock__: + return func(*args, **kwargs) - @property - def in_build_process(self): - return self._in_build_process + return lock_func class ProgramTranslator(object): @@ -260,7 +262,7 @@ class ProgramTranslator(object): import paddle.fluid as fluid - # Two motheds get same object because ProgramTranslator is a singleton + # Two methods get same object because ProgramTranslator is a singleton fluid.dygraph.ProgramTranslator() fluid.dygraph.ProgramTranslator.get_instance() @@ -289,22 +291,12 @@ class ProgramTranslator(object): cls._instance._initialized = False cls._instance.__init__() - def __init__(self, exe=None, place=None): + def __init__(self): # To make sure that calls __init__ only once. if self._initialized: return self._initialized = True - self._place = core.CPUPlace() if place is None else place - if exe is None: - self._exe = executor.Executor(self._place) - else: - self._exe = exe self._program_cache = ProgramCache() - self._optimizer_info = None - self._optimizer = None - self._loss_name = None - # Once startup_program is changed, should run startup_program. - self._prev_startup = None self.enable_declarative = True def enable(self, enable_declarative): @@ -384,24 +376,19 @@ class ProgramTranslator(object): assert callable( dygraph_func ), "Input dygraph_func is not a callable in ProgramTranslator.get_output" - if in_dygraph_mode() or not self.enable_declarative: + if not self.enable_declarative: logger.info( - "The ProgramTranslator.get_output doesn't work in dygraph " - "mode or set ProgramTranslator.enable to False. We will " - "just return dygraph output.") + "The ProgramTranslator.get_output doesn't work when setting ProgramTranslator.enable = False. " + "We will just return dygraph output.") return dygraph_func(*args, **kwargs) - program_cache = self.get_program_cache() - outputs = program_cache.build_program_and_return_output(dygraph_func, - *args, **kwargs) - if not program_cache.in_build_process: - outputs = self._run(*args, **kwargs) - with guard(): - if len(outputs) == 1: - outputs = to_variable(outputs[0]) - else: - outputs = tuple(to_variable(x) for x in outputs) - return outputs + function_spec = FunctionSpec(dygraph_func, args, kwargs) + _, partial_program_layer = self._program_cache[function_spec] + + if args and isinstance(args[0], layers.Layer): + args = args[1:] + + return partial_program_layer(args) def get_func(self, dygraph_func): """ @@ -440,10 +427,9 @@ class ProgramTranslator(object): assert callable( dygraph_func ), "Input dygraph_func is not a callable in ProgramTranslator.get_func" - if in_dygraph_mode() or not self.enable_declarative: + if not self.enable_declarative: logger.info( - "The ProgramTranslator.get_func doesn't work in dygraph " - "mode or set ProgramTranslator.enable to False. We will " + "The ProgramTranslator.get_func doesn't work when setting ProgramTranslator.enable=False. We will " "just return dygraph output.") return dygraph_func @@ -494,17 +480,28 @@ class ProgramTranslator(object): assert callable( dygraph_func ), "Input dygraph_func is not a callable in ProgramTranslator.get_program" - if in_dygraph_mode() or not self.enable_declarative: + if not self.enable_declarative: logger.info( - "The ProgramTranslator.get_program doesn't work in dygraph " - "mode or set ProgramTranslator.enable to False. We will " - "just return dygraph output.") + "The ProgramTranslator.get_program doesn't work when setting ProgramTranslator.enable=False." + "We will just return dygraph output.") return dygraph_func(*args, **kwargs) - program_cache = self.get_program_cache() - outputs = program_cache.build_program_and_return_output(dygraph_func, - *args, **kwargs) - return self.main_program, self.startup_program, program_cache.inputs, outputs + func_spec = FunctionSpec(dygraph_func, args, kwargs) + concrete_program, _ = self._program_cache[func_spec] + # Note: concrete_program hold all input/output infos include non-Variable + input_vars = [ + var for var in concrete_program.inputs + if isinstance(var, framework.Variable) + ] + output_vars = [ + var for var in concrete_program.outputs + if isinstance(var, framework.Variable) + ] + + return concrete_program.main_program, \ + concrete_program.startup_program, \ + input_vars, \ + output_vars def get_code(self, dygraph_func): """ @@ -552,73 +549,12 @@ class ProgramTranslator(object): source_code = ast_to_source_code(root_wrapper.node) return source_code - def _run(self, *args, **kwargs): - """ - Executes main_program and returns output Tensors. - """ - feed_dict, fetch_list = self._prepare(args) - - main_program = self._program_cache.main_program - outputs = self._exe.run(main_program, - feed=feed_dict, - fetch_list=fetch_list) - - return outputs - - def set_optimizer(self, optimizer, index_of_loss=0): - """ - Supports to set or update the optimizer used to minimize loss. - - Note: this method is an experimental API and may be changed in the near - future. - - Parameters: - optimizer (fluid optimizer): the training optimizer. - index_of_loss (int): the index of return variable as loss to be - minimized by optimizer. The default value is 0. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle.fluid as fluid - import numpy as np - - from paddle.fluid.dygraph.nn import Linear - - @fluid.dygraph.declarative - def linear_func(x): - x = fluid.dygraph.to_variable(x) - linear = Linear(32, 1) - y = linear(x) - z = linear(x) - return y, z - - prog_trans = fluid.dygraph.ProgramTranslator() - - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001) - prog_trans.set_optimizer(adam,index_of_loss=1) # minimize on 'z' - - for i in range(10): - y, z_loss = linear_func(np.ones(32).astype('float32')) - print(z_loss.numpy()) - - """ - check_type(index_of_loss, "index_of_loss", int, - "ProgramTranslator.set_optimizer") - self._check_cache_valid() - if self._optimizer and self._loss_name: - raise ValueError( - "{} for {} has already been set before. Please confirm not to call `set_optimizer` in for loop. ". - format(self._optimizer, self._loss_name)) - self._optimizer_info = (optimizer, index_of_loss) - def save_inference_model(self, dirname, feed=None, fetch=None): """ - Saves current model as the inference model. The saved - inference model can be loaded by C++ inference APIs. + Saves current model as the inference model. It will prune the main_program + to build a new program especially for inference, and then save it and all + related parameters to given `dirname` . The saved inference model can be + loaded by `:ref:`api_fluid_io_load_inference_model` or `C++ inference APIs. Args: dirname (str): the directory to save the inference model. @@ -630,151 +566,79 @@ class ProgramTranslator(object): saved inference model. If None, all output variables of the TracedLayer object would be the outputs of the saved inference model. Default None. - Returns: None - Examples: .. code-block:: python - - import paddle.fluid as fluid import numpy as np - - from paddle.fluid.dygraph.nn import Linear - - @fluid.dygraph.declarative - def linear_func(x): - x = fluid.dygraph.to_variable(x) - linear = Linear(32, 1) - y = linear(x) - z = linear(x) - return y, z - - - prog_trans = fluid.dygraph.ProgramTranslator() - - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001) - prog_trans.set_optimizer(adam,index_of_loss=1) # minimize on 'z' - - for i in range(10): - y, z_loss = linear_func(np.ones(32).astype('float32')) - print(z_loss.numpy()) - + import paddle.fluid as fluid + from paddle.fluid.dygraph import Linear + from paddle.fluid.dygraph import declarative + from paddle.fluid.dygraph import ProgramTranslator + + class SimpleNet(fluid.dygraph.Layer): + def __init__(self, in_size, out_size): + super(SimpleNet, self).__init__() + self._linear = Linear(in_size, out_size) + + @declarative + def forward(self, x): + y = self._linear(x) + z = self._linear(y) + loss = fluid.layers.mean(z) + return z, loss + + with fluid.dygraph.guard(fluid.CPUPlace()): + net = SimpleNet(8, 8) + adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) + x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + for i in range(10): + loss, out = net(x) + loss.backward() + adam.minimize(loss) + net.clear_gradients() # Save inference model. # Note that fetch=[0] means we set 'y' as the inference output. + prog_trans = ProgramTranslator() prog_trans.save_inference_model("./dy2stat_infer_model", fetch=[0]) # In this example, the inference model will be pruned based on input (x) and # output (y). The pruned inference program is going to be saved in the folder # "./dy2stat_infer_model" and parameters are going to be saved in separate # files in the folder. - """ - program_cache = self.get_program_cache() - if feed is None: - feeded_var_names = [i.name for i in program_cache.inputs] - else: - feeded_var_names = [program_cache.inputs[i].name for i in feed] - if fetch is None: - fetch_vars = program_cache.outputs - else: - fetch_vars = [program_cache.outputs[i] for i in fetch] - from paddle.fluid.io import save_inference_model - save_inference_model( - dirname=dirname, - feeded_var_names=feeded_var_names, - target_vars=fetch_vars, - executor=self._exe, - main_program=self.main_program.clone()) - - def _prepare(self, args): - """ - Prepares with feed_dict, fetch_list, optimizer and initialize vars - by running startup_program. - """ + def get_feed_fetch(var_list, partial_vars, return_name=False): + vars = [ + var for var in var_list if isinstance(var, framework.Variable) + ] + if partial_vars: + vars = [vars[idx] for idx in partial_vars] + if return_name: + vars = [var.name for var in vars] - # Updates batch_data for feed_dict - feed_dict = self._update_batch_data(args) - fetch_list = self._program_cache.outputs + return vars - # Adds optimizer if needed. - if self._optimizer_info and self._optimizer is None: - self._add_optimizer() - - if self._need_startup(): - self._exe.run(self.startup_program) - self._prev_startup = self.startup_program - - return feed_dict, fetch_list - - def _need_startup(self): - """ - Determines whether needy to run startup_program. - """ - if self.startup_program != self._prev_startup: - check_type(self.startup_program, "startup_program", - framework.Program, "_need_startup") - return len(self.startup_program.global_block().ops) > 0 - - return False - - def _check_cache_valid(self): - """ - Checks whether the current program is consistent with `default_main_program`. - In some models and unittest, program will be switched frequently by `program_guard`. - If does, the cached program and other properties are not available and should be reset. - """ - if self._program_cache.main_program: - if self._program_cache.main_program != framework.default_main_program( - ): - ProgramTranslator.reset() - - def _update_batch_data(self, args): - """ - Updates cached batch data while training program. - """ - feed_name_to_idx = self._program_cache.feed_name_to_idx - feed_vars = self._program_cache.inputs - feed_dict = {} - for feed_var in feed_vars: - idx = feed_name_to_idx[feed_var.name] - feed_dict[feed_var.name] = args[idx] + func_spec, (concrete_program, + partial_layer) = self._program_cache.last() + # share paramBase data with parameter + scope = core.Scope() + for param_base in concrete_program.parameters: + param_tensor = scope.var(param_base.name).get_tensor() + src_tensor = param_base.value().get_tensor() + param_tensor._share_data_with(src_tensor) - return feed_dict + feed_var_names = get_feed_fetch(concrete_program.inputs, feed, True) + fetch_vars = get_feed_fetch(concrete_program.outputs, fetch) - def _add_optimizer(self): - """ - Supports to set or update the optimizer used to minimize loss. - """ - optimizer, index_of_loss = self._optimizer_info - - outputs = self._program_cache.outputs - outputs = [outputs] if not isinstance(outputs, - (list, tuple)) else outputs - - assert abs(index_of_loss) < len(outputs), \ - "index_of_loss: {} shall not exceed the length of outputs: {}.".format( - index_of_loss, len(outputs)) - - loss_var = outputs[index_of_loss] - check_type(loss_var, "loss_var", framework.Variable, - "ProgramTranslator._add_optimizer") - - main_program = self._program_cache.main_program - startup_program = self._program_cache.startup_program - all_vars = main_program.block(0).vars - - if all_vars.get(loss_var.name, None) is None: - raise ValueError( - "Can't find {} in main_program, please confirm whether the input loss is correct." - .format(loss_var.name)) - # Adds optimizer to minimize loss - with framework.program_guard(main_program, startup_program): - optimizer.minimize(loss_var) - - self._optimizer = optimizer - self._loss_name = loss_var.name + from paddle.fluid.io import save_inference_model + with scope_guard(scope): + save_inference_model( + dirname=dirname, + feeded_var_names=feed_var_names, + target_vars=fetch_vars, + executor=executor.Executor(framework._current_expected_place()), + main_program=concrete_program.main_program.clone()) def get_program_cache(self): """ @@ -787,20 +651,11 @@ class ProgramTranslator(object): Examples: .. code-block:: python - + import paddle.fluid as fluid prog_trans = fluid.dygraph.ProgramTranslator() prog_cache = prog_trans.get_program_cache() """ - self._check_cache_valid() return self._program_cache - - @property - def main_program(self): - return self._program_cache.main_program - - @property - def startup_program(self): - return self._program_cache.startup_program diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index c4bbd278ac..44641b3bd6 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -54,6 +54,9 @@ class NodeVarType(object): # We use this enum value to denote the type return by a Paddle API PADDLE_RETURN_TYPES = 304 + # If node.node_var_type in TENSOR_TYPES, it can be considered as tensor-dependent. + TENSOR_TYPES = {TENSOR, PADDLE_RETURN_TYPES} + @staticmethod def binary_op_output_type(in_type1, in_type2): if in_type1 == in_type2: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py index 52392bc1e0..a21a5af455 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py @@ -15,15 +15,8 @@ from __future__ import print_function import gast -import astor -import copy -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType, StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.utils import is_control_flow_to_transform -from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api, is_dygraph_api, is_to_variable -from paddle.fluid.dygraph.dygraph_to_static.utils import to_assign_node, to_static_ast, update_args_of_func -from paddle.fluid.dygraph.dygraph_to_static.utils import dygraph_class_to_static_api, create_api_shape_node +from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api +from paddle.fluid.dygraph.dygraph_to_static.utils import create_api_shape_node from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, NodeVarType from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 5a9146420d..217d488aba 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -50,8 +50,9 @@ def is_api_in_module(node, module_prefix): # source_file = inspect.getfile(dyfunc) # import_statements = ImportVisitor(source_file).transform() # import_str = "".join(import_statements) - import paddle.fluid as fluid import paddle + import paddle.fluid as fluid + import paddle.fluid.layers as layers from paddle.fluid.dygraph import to_variable import paddle.fluid.dygraph as dygraph return eval("_is_api_in_module_helper({}, '{}')".format(func_str, @@ -88,29 +89,19 @@ def is_numpy_api(node): return False -def is_control_flow_to_transform(node, var_name_to_type): +def is_control_flow_to_transform(node, + static_analysis_visitor=None, + var_name_to_type=None): """ - Determines whether the node is a Paddle control flow statement which needs to - transform into a static graph control flow statement. + Determines whether the node is a PaddlePaddle control flow statement which needs to + be transformed into a static graph control flow statement. """ assert isinstance(node, gast.AST), \ "The type of input node must be gast.AST, but received %s." % type(node) - - if isinstance(node, gast.If): - from .ifelse_transformer import IfConditionVisitor - if_visitor = IfConditionVisitor( - node.test, node_var_type_map=var_name_to_type) - return if_visitor.is_control_flow() - - if isinstance(node, gast.For): - # TODO: make a better condition - return True - - if isinstance(node, gast.While): - # TODO: make a better condition - return True - - return False + visitor = IsControlFlowVisitor( + node, static_analysis_visitor, node_var_type_map=var_name_to_type) + need_to_transform = visitor.transform() + return need_to_transform def _delete_keywords_from(node): @@ -415,3 +406,208 @@ def ast_to_source_code(ast_node): ast_node = gast.gast_to_ast(ast_node) source_code = astor.to_source(ast_node) return source_code + + +def is_candidate_node(node): + """ + Nodes with specified type will be dependent on tensor. + """ + is_compare_node = isinstance(node, (gast.Compare, gast.BoolOp, gast.UnaryOp, + gast.For, gast.If, gast.While)) + # TODO(Aurelius84): `.numpy()` may be an customized function, + # and should consider a more elegant way to solve this problem. + has_numpy_attr = ".numpy()" in ast_to_source_code(node) + return is_compare_node or has_numpy_attr + + +def compare_with_none(node): + """ + Whether the comparator of `gast.Compare` node is `None`. + """ + if isinstance(node, gast.Compare): + for child in [node.left, node.comparators]: + # node.comparators is a list. + if isinstance(child, list): + child = child[0] + if (isinstance(child, gast.Constant) and child.value is None) or ( + isinstance(child, gast.Name) and child.id == 'None'): + return True + return False + + +class IsControlFlowVisitor(gast.NodeVisitor): + """ + Judge whether the ast_node of control flow from Dygraph code dependent on paddle Tensor. + `ast_node` can be gast.If, gast.For, gast.While, gast.If.test(gast.Compare, gast.BoolOp, gast.UnaryOp). + + If returns True, + gast.If.test must meet at least one of the following requirements: + 1. involves at least one var whose type is Tensor. + 2. the Tensor var calls `.numpy()[]` interface or Tensor.shape is [1]. + 3. involves Tensor.shape[i] and the shape[i] is unknown in compile time. + gast.While must meet at least one of the requirements 1 to 5: + 4. has `break` statement. + 5. has `continue` statement. + gast.For must meet at least one of the requirements 4 to 6: + 6. calls `range` function in `for` statement and the argument of range is Tensor. + TODO: Support non-range case + + The following examples should not be considered as control_flow_if: + 1. `if Tensor_var` or `if Tensor_var is None` + 2. if Tensor.shape[i] is determined with fixed value (not -1 or None) + + Note: pred in ConditionalBlock require variable, which means all vars should be Tensor + or transformed into Tensor, like fill_constant(shape=[1], dtype='int32', value=Tensor.shape[i]). + + TODO: 1. need to deal with `tensor.shape[i]` which need to eval the data of shape[i], + because reshape_op may be called before this statement. + """ + + def __init__(self, + ast_node, + static_analysis_visitor=None, + node_var_type_map=None): + assert isinstance( + ast_node, gast.AST + ), "Type of input node should be gast.AST, but received %s." % type( + ast_node) + self.ast_root = ast_node + if static_analysis_visitor is None: + from .static_analysis import StaticAnalysisVisitor + static_analysis_visitor = StaticAnalysisVisitor(ast_node) + self.static_analysis_visitor = static_analysis_visitor + self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + ) + self.node_var_type_map = node_var_type_map + + self.is_control_flow_num = 0 + self._compare_node_tenor_set = set() + + def transform(self): + node = self.ast_root + if isinstance(node, gast.If): + self._visit_If(node) + elif isinstance(node, gast.For): + self._visit_For(node) + elif isinstance(node, gast.While): + self._visit_While(node) + else: + self.visit(node) + return self.is_control_flow_num > 0 + + def _visit_If(self, node): + assert isinstance(node, gast.If) + self.visit(node.test) + return + + def _visit_For(self, node): + assert isinstance(node, gast.For) + # TODO + # self.is_control_flow_num += 1 + if not isinstance(node.iter, gast.Call): + return + if not isinstance(node.iter.func, gast.Name): + return + if node.iter.func.id != "range": + return + for arg in node.iter.args: + self.visit(arg) + + for child_node in gast.walk(node): + if isinstance(child_node, (gast.Continue, gast.Break)): + self._visit_break_continue(child_node) + return + + def _visit_While(self, node): + assert isinstance(node, gast.While) + test = node.test + self.generic_visit(test) + for child_node in gast.walk(node): + if isinstance(child_node, (gast.Continue, gast.Break)): + self._visit_break_continue(child_node) + return + + def _visit_break_continue(self, node): + assert isinstance(node, (gast.Break, gast.Continue)) + wrapper_node = self.node_to_wrapper_map.get(node) + if not wrapper_node: + # Transformed node is not in node_to_wrapper_map + return + + while wrapper_node.parent: + parent_node = wrapper_node.parent.node + if isinstance(parent_node, (gast.For, gast.While)): + if parent_node is self.ast_root: + self.is_control_flow_num += 1 + return + else: + return + + wrapper_node = wrapper_node.parent + + return + + def visit_BoolOp(self, node): + for i, child in enumerate(node.values): + self.visit(child) + return node + + def visit_Compare(self, node): + pre_control_flow_num = self.is_control_flow_num + if not compare_with_none(node): + self.generic_visit(node) + for child in gast.walk(node): + if isinstance(child, gast.Subscript): + self._visit_Subscript(child) + if self.is_control_flow_num > pre_control_flow_num: + self._compare_node_tenor_set.add(node) + return node + + def _visit_Subscript(self, node): + self.generic_visit(node) + if hasattr(node, 'value') and isinstance(node.value, gast.Call): + self._visit_Call(node.value) + return node + + def _visit_Call(self, node): + assert isinstance(node, gast.Call) + if isinstance(node.func, gast.Attribute): + attr_node = node.func + if attr_node.attr == 'numpy': + self.is_control_flow_num += 1 + + def visit_Call(self, node): + self._visit_Call(node) + if is_paddle_api(node): + self.is_control_flow_num += 1 + return node + + def visit_Name(self, node): + if self._is_node_with_tensor(node, node.id): + self.is_control_flow_num += 1 + return node + + def visit_Constant(self, node): + if self._is_node_with_tensor(node, node.value): + self.is_control_flow_num += 1 + return node + + def _is_node_with_tensor(self, node, name_id): + from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType + + # Look up the node_var_type_map by name_id. + if self.node_var_type_map: + if name_id and isinstance(name_id, six.string_types): + var_type = self.node_var_type_map.get(name_id, None) + if var_type and var_type & NodeVarType.TENSOR_TYPES: + return True + # if not found, look up the node_to_wrapper_map by node. + wrapper_node = self.node_to_wrapper_map.get(node, None) + if wrapper_node is not None: + if wrapper_node.node_var_type & NodeVarType.TENSOR_TYPES: + return True + + return False + + def get_compare_nodes_with_tensor(self): + return self._compare_node_tenor_set diff --git a/python/paddle/fluid/dygraph/jit.py b/python/paddle/fluid/dygraph/jit.py index 7343bed130..8da05f89c9 100644 --- a/python/paddle/fluid/dygraph/jit.py +++ b/python/paddle/fluid/dygraph/jit.py @@ -17,7 +17,6 @@ from __future__ import print_function __all__ = ['TracedLayer', 'declarative', 'dygraph_to_static_func'] import logging - from paddle.fluid import core from paddle.fluid.compiler import CompiledProgram from paddle.fluid.dygraph.base import program_desc_tracing_guard, switch_to_static_graph @@ -156,13 +155,11 @@ def _declarative_(dygraph_func): def __impl__(*args, **kwargs): program_translator = ProgramTranslator() - if in_dygraph_mode() or not program_translator.enable_declarative: + if not program_translator.enable_declarative: logger.info( - "The decorator 'declarative' doesn't work in dygraph " - "mode or set ProgramTranslator.enable to False. We will " - "just return dygraph output.") + "The decorator 'declarative' doesn't work when setting ProgramTranslator.enable=False. " + "We will just return dygraph output.") return dygraph_func(*args, **kwargs) - program_translator = ProgramTranslator() return program_translator.get_output(dygraph_func, *args, **kwargs) return __impl__ @@ -228,6 +225,7 @@ class TracedLayer(object): self._program = program self._feed_names = feed_names self._fetch_names = fetch_names + self._params = parameters self._place = _current_expected_place() diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index c6327a7fae..d7572cfe1e 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -23,7 +23,7 @@ from . import parallel_helper from .. import unique_name from paddle.fluid import core from .layer_object_helper import LayerObjectHelper -from .base import program_desc_tracing_guard +from .base import program_desc_tracing_guard, param_guard from paddle.fluid import framework from ..param_attr import ParamAttr import copy @@ -457,7 +457,8 @@ class Layer(core.Layer): self._parameters.values()) self._built = True - outputs = self.forward(*inputs, **kwargs) + with param_guard(self._parameters): + outputs = self.forward(*inputs, **kwargs) for forward_post_hook in self._forward_post_hooks.values(): hook_result = forward_post_hook(self, inputs, outputs) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 5482e6e2f2..d88ba16b76 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -1915,7 +1915,7 @@ class Operator(object): in_arg_names.append(arg) elif isinstance(arg, six.binary_type): in_arg_names.append(arg.decode()) - elif isinstance(arg, Variable): + elif isinstance(arg, (Variable, core.VarBase)): in_arg_names.append(cpt.to_text(arg.name)) else: raise TypeError( diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py index 99f5e81fd9..15bd823713 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py @@ -15,7 +15,6 @@ from __future__ import print_function import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func def add_fn(x): @@ -142,7 +141,6 @@ class NetWithControlFlowIf(fluid.dygraph.Layer): self.alpha = 10. self.constant_vars = {} - @dygraph_to_static_func def forward(self, input): hidden_dim = input.shape[-1] if hidden_dim != self.hidden_dim: @@ -250,3 +248,31 @@ def if_with_class_var(x, y=None): else: x = x - foo.b return x + + +def if_tensor_case(x): + x = fluid.dygraph.to_variable(x) + + mean = fluid.layers.mean(x) + # It is equivalent to `if mean != 0` + if mean: + for i in range(0, 10): + if i > 5: + x += 1 + break + x += 1 + else: + for i in range(0, 37): + x += 1 + break + x += i + + # join `and`/`or` + if fluid.layers.mean(x) + 1 and mean > 1 and x is not None or 2 > 1: + x -= 1 + + # `not` statement + if not (x[0][0] and (mean * x)[0][0]): + x += 1 + + return x diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py index 5c91f08f32..9a605078fb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py @@ -17,7 +17,7 @@ from __future__ import print_function import unittest import numpy as np import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph.jit import declarative SEED = 2020 np.random.seed(SEED) @@ -160,13 +160,9 @@ class TestContinueInFor(unittest.TestCase): return res.numpy() def run_static_mode(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - res = dygraph_to_static_func(self.dygraph_func)(self.input) - exe = fluid.Executor(self.place) - static_res = exe.run(main_program, fetch_list=[res]) - - return static_res[0] + with fluid.dygraph.guard(): + res = declarative(self.dygraph_func)(self.input) + return res.numpy() def test_transformed_static_result(self): static_res = self.run_static_mode() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py index 92e36aa8d7..192e1a6b11 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py @@ -36,8 +36,7 @@ class TestCacheProgram(unittest.TestCase): def test_cache(self): prev_ops, cur_ops = Counter(), Counter() prev_out, cur_out = None, None - main_program = fluid.Program() - with fluid.program_guard(main_program): + with fluid.dygraph.guard(fluid.CPUPlace()): static_net = self.dygraph_class() for batch_id in range(self.batch_num): out = static_net(self.data) @@ -51,9 +50,9 @@ class TestCacheProgram(unittest.TestCase): ]) if batch_id > 0: prev_out_numpy = prev_out[0].numpy() if isinstance( - prev_out, tuple) else prev_out.numpy() + prev_out, (tuple, list)) else prev_out.numpy() cur_out_numpy = cur_out[0].numpy() if isinstance( - cur_out, tuple) else cur_out.numpy() + cur_out, (tuple, list)) else cur_out.numpy() self.assertTrue( np.allclose(prev_out_numpy, cur_out_numpy), msg='Output in previous batch is {}\n Output in current batch is \n{}' @@ -75,29 +74,23 @@ class TestCacheProgramWithOptimizer(unittest.TestCase): self.batch_num = 5 def train_static(self): - main_program = fluid.Program() - loss_data = [] - with fluid.program_guard(main_program): - static_net = self.dygraph_class() - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001) - # set optimizer - program_translator = ProgramTranslator() - program_translator.set_optimizer(adam, index_of_loss=1) + return self.train(to_static=True) - for batch_id in range(self.batch_num): - pred, avg_loss = static_net(self.data) - loss_data.append(np.array(avg_loss.numpy())) + def train_dygraph(self): + return self.train(to_static=False) - return loss_data + def train(self, to_static=False): + prog_trans = ProgramTranslator() + prog_trans.enable(to_static) - def train_dygraph(self): with fluid.dygraph.guard(fluid.CPUPlace()): dygraph_net = self.dygraph_class() adam = fluid.optimizer.AdamOptimizer( learning_rate=0.001, parameter_list=dygraph_net.parameters()) loss_data = [] for batch_id in range(self.batch_num): - pred, avg_loss = dygraph_net(self.data) + input = fluid.dygraph.to_variable(self.data) + pred, avg_loss = dygraph_net(input) loss_data.append(avg_loss.numpy()) avg_loss.backward() @@ -114,20 +107,6 @@ class TestCacheProgramWithOptimizer(unittest.TestCase): msg='dygraph is {}\n static_res is \n{}'.format(dygraph_loss, static_loss)) - def test_exception(self): - main_program = fluid.Program() - loss_data = [] - with fluid.program_guard(main_program): - static_net = self.dygraph_class() - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.001) - # set optimizer - program_translator = ProgramTranslator() - - with self.assertRaisesRegexp(ValueError, "has already been set"): - for batch_id in range(self.batch_num): - program_translator.set_optimizer(adam, index_of_loss=1) - static_net(self.data) - def simple_func(x): inputs = fluid.dygraph.to_variable(x) @@ -156,7 +135,6 @@ def sum_even_util_limit(max_len, limit): return ret_sum -@declarative def sum_under_while(limit): i = fluid.dygraph.to_variable(np.zeros((1)).astype('int32')) ret_sum = fluid.dygraph.to_variable(np.zeros((1)).astype('int32')) @@ -168,11 +146,12 @@ def sum_under_while(limit): class TestToOutputWithCache(unittest.TestCase): def test_output(self): - ret = sum_even_util_limit(80, 10) - self.assertEqual(ret.numpy(), 30) + with fluid.dygraph.guard(): + ret = sum_even_util_limit(80, 10) + self.assertEqual(ret.numpy(), 30) - ret = sum_under_while(100) - self.assertEqual(ret.numpy(), 5050) + ret = declarative(sum_under_while)(100) + self.assertEqual(ret.numpy(), 5050) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py index 79c63ea601..c8051b3f24 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py @@ -19,7 +19,8 @@ import numpy as np import unittest import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator PLACE = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace( ) @@ -75,7 +76,7 @@ class MainNetWithDict(fluid.dygraph.Layer): self.output_size = output_size self.sub_net = SubNetWithDict(hidden_size, output_size) - @dygraph_to_static_func + @declarative def forward(self, input, max_len=4): input = fluid.dygraph.to_variable(input) cache = { @@ -121,17 +122,14 @@ class TestNetWithDict(unittest.TestCase): self.batch_size = self.x.shape[0] def _run_static(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - net = MainNetWithDict(batch_size=self.batch_size) - # Transform into static graph - out = net(self.x) - exe = fluid.Executor(PLACE) - exe.run(fluid.default_startup_program()) - ret = exe.run(main_program, fetch_list=out) - return ret[0] + return self.train(to_static=True) def _run_dygraph(self): + return self.train(to_static=False) + + def train(self, to_static=False): + prog_trans = ProgramTranslator() + prog_trans.enable(to_static) with fluid.dygraph.guard(PLACE): net = MainNetWithDict(batch_size=self.batch_size) ret = net(self.x) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py index 5622480332..146608cb07 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py @@ -14,12 +14,12 @@ from __future__ import print_function -from paddle.fluid.dygraph.jit import declarative - import numpy as np import unittest import paddle.fluid as fluid +from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator SEED = 2020 @@ -32,22 +32,20 @@ class Pool2D(fluid.dygraph.Layer): @declarative def forward(self, x): - inputs = fluid.dygraph.to_variable(x) - # Add func `get_result` for testing arg_name_to_idx in ast transformation. def get_result(x): return self.pool2d(x) - pre = get_result(inputs) + pre = get_result(x) return pre class Linear(fluid.dygraph.Layer): - def __init__(self): + def __init__(self, input_dim=10, output_dim=5): super(Linear, self).__init__() self.fc = fluid.dygraph.Linear( - input_dim=10, - output_dim=5, + input_dim, + output_dim, act='relu', param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( value=0.99)), @@ -56,8 +54,7 @@ class Linear(fluid.dygraph.Layer): @declarative def forward(self, x): - inputs = fluid.dygraph.to_variable(x) - pre = self.fc(inputs) + pre = self.fc(x) loss = fluid.layers.mean(pre) return pre, loss @@ -67,28 +64,28 @@ class TestPool2D(unittest.TestCase): self.dygraph_class = Pool2D self.data = np.random.random((1, 2, 4, 4)).astype('float32') - def run_dygraph_mode(self): + def train(self, to_static=False): + program_translator = ProgramTranslator() + program_translator.enable(to_static) + with fluid.dygraph.guard(): dy_layer = self.dygraph_class() - prediction = dy_layer(x=self.data) + x = fluid.dygraph.to_variable(self.data) + prediction = dy_layer(x) if isinstance(prediction, (list, tuple)): prediction = prediction[0] return prediction.numpy() - def run_static_mode(self): - startup_prog = fluid.Program() - main_prog = fluid.Program() - with fluid.program_guard(main_prog, startup_prog): - dy_layer = self.dygraph_class() - out = dy_layer(x=self.data) - if isinstance(out, tuple): - return out[0].numpy() - return out.numpy() - - def test_static_output(self): - dygraph_res = self.run_dygraph_mode() - static_res = self.run_static_mode() + def train_static(self): + return self.train(to_static=True) + + def train_dygraph(self): + return self.train(to_static=False) + + def test_declarative(self): + dygraph_res = self.train_dygraph() + static_res = self.train_static() self.assertTrue( np.allclose(dygraph_res, static_res), diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py index 9724e6d936..4f7fa65ee9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py @@ -17,8 +17,7 @@ from __future__ import print_function import numpy as np import paddle.fluid as fluid import unittest - -from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph import declarative @fluid.dygraph.declarative @@ -31,7 +30,7 @@ def dygraph_decorated_func(x): return x_v -@fluid.dygraph.jit.declarative +@fluid.dygraph.declarative def jit_decorated_func(x): x = fluid.dygraph.to_variable(x) if fluid.layers.mean(x) > 0: @@ -62,18 +61,14 @@ class TestFullNameDecorator(unittest.TestCase): def test_run_success(self): x = np.ones([1, 2]).astype("float32") answer = np.zeros([1, 2]).astype("float32") - with fluid.program_guard(fluid.Program(), fluid.Program()): + with fluid.dygraph.guard(): self.assertTrue( np.allclose(dygraph_decorated_func(x).numpy(), answer)) - with fluid.program_guard(fluid.Program(), fluid.Program()): self.assertTrue(np.allclose(jit_decorated_func(x).numpy(), answer)) - with fluid.program_guard(fluid.Program(), fluid.Program()): self.assertTrue( np.allclose(decorated_call_decorated(x).numpy(), answer)) - with fluid.program_guard(fluid.Program(), fluid.Program()): with self.assertRaises(NotImplementedError): DoubleDecorated().double_decorated_func1(x) - with fluid.program_guard(fluid.Program(), fluid.Program()): with self.assertRaises(NotImplementedError): DoubleDecorated().double_decorated_func2(x) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py index abac936f65..0363e32e03 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py @@ -15,10 +15,10 @@ from __future__ import print_function import numpy as np -import paddle.fluid as fluid import unittest -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator from ifelse_simple_func import * @@ -41,19 +41,16 @@ class TestDygraphIfElse(unittest.TestCase): self.dyfunc = dyfunc_with_if_else def _run_static(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - x_v = fluid.layers.assign(self.x) - # Transform into static graph - out = dygraph_to_static_func(self.dyfunc)(x_v) - exe = fluid.Executor(place) - ret = exe.run(main_program, fetch_list=out) - return ret + return self._run_dygraph(to_static=True) + + def _run_dygraph(self, to_static=False): - def _run_dygraph(self): with fluid.dygraph.guard(place): x_v = fluid.dygraph.to_variable(self.x) - ret = self.dyfunc(x_v) + if to_static: + ret = declarative(self.dyfunc)(x_v) + else: + ret = self.dyfunc(x_v) return ret.numpy() def test_ast_to_func(self): @@ -176,6 +173,12 @@ class TestDygraphIfElseWithClassVar(TestDygraphIfElse): self.dyfunc = if_with_class_var +class TestDygraphIfTensor(TestDygraphIfElse): + def setUp(self): + self.x = np.random.random([10, 16]).astype('float32') + self.dyfunc = if_tensor_case + + class TestDygraphIfElseNet(unittest.TestCase): """ TestCase for the transformation from control flow `if/else` @@ -187,18 +190,15 @@ class TestDygraphIfElseNet(unittest.TestCase): self.Net = NetWithControlFlowIf def _run_static(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - net = self.Net() - x_v = fluid.layers.assign(self.x) - # Transform into static graph - out = net(x_v) - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - ret = exe.run(main_program, fetch_list=out) - return ret[0] + return self._run(to_static=True) def _run_dygraph(self): + return self._run(to_static=False) + + def _run(self, to_static=False): + prog_trans = ProgramTranslator() + prog_trans.enable(to_static) + with fluid.dygraph.guard(place): net = self.Net() x_v = fluid.dygraph.to_variable(self.x) @@ -234,7 +234,7 @@ class TestAst2FuncWithExternalFunc(TestDygraphIfElse): class NetWithExternalFunc(fluid.dygraph.Layer): - @dygraph_to_static_func + @declarative def forward(self, x, label=None): if fluid.layers.mean(x) < 0: x_v = x - 1 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py index e40fa00ddf..1efa4961b7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py @@ -19,9 +19,9 @@ import textwrap import gast from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import get_name_ids from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfConditionVisitor -from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IsControlFlowVisitor from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType +from paddle.fluid.dygraph.dygraph_to_static.utils import IsControlFlowVisitor class TestGetNameIds(unittest.TestCase): @@ -155,7 +155,18 @@ class TestIsControlFlowIf(unittest.TestCase): self.check_false_case("fluid.layers.sum(x).numpy() != None") def test_is_None4(self): - self.check_false_case("fluid.layers.sum(x) and 2>1") + node = gast.parse("fluid.layers.sum(x) and 2>1") + node_test = node.body[0].value + + if_visitor = IfConditionVisitor(node_test) + self.assertTrue(if_visitor.is_control_flow()) + # Transformation result: + # bool_tensor_0 = fluid.layers.cast(x=fluid.layers.sum(x), dtype='bool') + # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1)) + # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=bool_tensor_1) + + new_node, assign_nodes = if_visitor.transform() + self.assertTrue(len(assign_nodes) == 3) def test_if(self): node = gast.parse("x.numpy()[1] > 1") @@ -253,34 +264,38 @@ class TestIsControlFlowIf(unittest.TestCase): self.assertTrue(len(assign_nodes) == 0) def test_paddle_api_with_andOr(self): - code = """ + code_or = """ def foo(x): if 2 > 1 and fluid.layers.shape(x)[0] > 16 or x is not None : x = x + 1 return x """ - code = """ + + code_and = """ def foo(x): if 2 > 1 and fluid.layers.shape(x)[0] > 16 and x is not None : x = x + 1 return x """ - code = textwrap.dedent(code) - node = gast.parse(code) - static_analysis_visitor = StaticAnalysisVisitor(node) - test_node = node.body[0].body[0].test - if_visitor = IfConditionVisitor(test_node, static_analysis_visitor) - self.assertTrue(if_visitor.is_control_flow()) - - new_node, assign_nodes = if_visitor.transform() - # Tranformation result: - # bool_tensor_0 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1)) - # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(x is not None)) - # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=fluid.layers.shape(x)[0] > 16) - # logic_and_1 = fluid.layers.logical_and(x=logic_and_0, y=bool_tensor_1) - - self.assertTrue(isinstance(new_node, gast.Name)) - self.assertTrue(len(assign_nodes) == 4) + for code in [code_or, code_and]: + code = textwrap.dedent(code) + node = gast.parse(code) + static_analysis_visitor = StaticAnalysisVisitor(node) + test_node = node.body[0].body[0].test + if_visitor = IfConditionVisitor(test_node, static_analysis_visitor) + self.assertTrue(if_visitor.is_control_flow()) + + new_node, assign_nodes = if_visitor.transform() + # Transformation result: + # bool_tensor_0 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(2 > 1)) + # bool_tensor_1 = fluid.layers.fill_constant(shape=[1], dtype='bool', value=bool(x is not None)) + # logic_and_0 = fluid.layers.logical_and(x=bool_tensor_0, y=fluid.layers.shape(x)[0] > 16) + + # logic_and_1 = fluid.layers.logical_and(x=logic_and_0, y=bool_tensor_1) for code_and + # logic_or_0= fluid.layers.logical_or(x=logic_and_0, y=bool_tensor_1) for code_and + + self.assertTrue(isinstance(new_node, gast.Name)) + self.assertTrue(len(assign_nodes) == 4) def test_with_node_var_type_map(self): node = gast.parse("x > 1") diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py index 588d900f31..3e65492524 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py @@ -47,6 +47,10 @@ def test_list_in_if(x): def test_list_in_for_loop(x, iter_num): x = fluid.dygraph.to_variable(x) + # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) # TODO(liym27): Delete it if the type of parameter iter_num can be resolved a = [] for i in range(iter_num): a.append(x) @@ -56,6 +60,10 @@ def test_list_in_for_loop(x, iter_num): def test_list_in_for_loop_with_concat(x, iter_num): x = fluid.dygraph.to_variable(x) a = [] + # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) # TODO(liym27): Delete it if the type of parameter iter_num can be resolved for i in range(iter_num): a.append(x) a = fluid.layers.concat(a, axis=0) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py index b64fa34500..66f153d9ef 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py @@ -20,8 +20,8 @@ import numpy as np import paddle.fluid as fluid import unittest -from paddle.fluid.dygraph.jit import dygraph_to_static_func from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import NameVisitor +from paddle.fluid.dygraph.jit import declarative SEED = 2020 np.random.seed(SEED) @@ -29,6 +29,9 @@ np.random.seed(SEED) def while_loop_dyfunc(x): i = fluid.dygraph.to_variable(x) + # Use `to_variable` so that static analysis can analyze the type of X is Tensor + x = fluid.dygraph.to_variable( + x) # TODO(liym27): Delete it if the type of parameter x can be resolved while x < 10: i = i + x x = x + 1 @@ -37,6 +40,9 @@ def while_loop_dyfunc(x): def while_loop_dyfun_with_conflict_var(x): i = fluid.dygraph.to_variable(x) + # Use `to_variable` so that static analysis can analyze the type of X is Tensor + x = fluid.dygraph.to_variable( + x) # TODO(liym27): Delete it if the type of parameter x can be resolved def relu(y): # 'y' is not visible outside the scope. @@ -56,6 +62,9 @@ def while_loop_dyfunc_with_none(x): i = fluid.dygraph.to_variable(x)\ if x is not None \ else fluid.dygraph.to_variable(x+1) + # Use `to_variable` so that static analysis can analyze the type of X is Tensor + x = fluid.dygraph.to_variable( + x) # TODO(liym27): Delete it if the type of parameter x can be resolved flag = 1 while x < 10: i = i + x if flag is not None else x + i @@ -72,6 +81,10 @@ def for_loop_dyfunc(max_len): def while_loop_bool_op(x): i = fluid.dygraph.to_variable(x) + + # Use `to_variable` so that static analysis can analyze the type of X is Tensor + x = fluid.dygraph.to_variable( + x) # TODO(liym27): Delete it if the type of parameter x can be resolved while (x >= 0 and x < 10) or x <= -1 or x < -3 or (x < -7 or x < -5): i = i + x x = x + 1 @@ -102,6 +115,11 @@ def for_loop_class_var(max_len): self.c = 5 foo = Foo() + + # Use `to_variable` so that static analysis can analyze the type of X is Tensor + # TODO(liym27): Delete it if the type of parameter x can be resolved + max_len = fluid.layers.fill_constant( + shape=[1], value=max_len, dtype="int32") for i in range(max_len): foo.b = fluid.layers.zeros(shape=[1], dtype='float32') foo.c = foo.b + foo.a @@ -149,19 +167,17 @@ class TestTransformWhileLoop(unittest.TestCase): self.dyfunc = while_loop_dyfunc def _run_static(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - x_var = fluid.layers.assign(self.x) - static_func = dygraph_to_static_func(self.dyfunc) - - out = static_func(x_var) - exe = fluid.Executor(self.place) - ret = exe.run(main_program, fetch_list=out) - return ret + return self._run(to_static=True) def _run_dygraph(self): + return self._run(to_static=False) + + def _run(self, to_static): with fluid.dygraph.guard(self.place): - ret = self.dyfunc(fluid.dygraph.to_variable(self.x)) + if to_static: + ret = declarative(self.dyfunc)(self.x) + else: + ret = self.dyfunc(self.x) return ret.numpy() def test_ast_to_func(self): @@ -201,22 +217,20 @@ class TestTransformForLoop(unittest.TestCase): self.dyfunc = for_loop_dyfunc def _run_static(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - static_func = dygraph_to_static_func(self.dyfunc) - out = static_func(self.len) - exe = fluid.Executor(self.place) - ret = exe.run(main_program, fetch_list=out) - return ret + return self._run(to_static=True) def _run_dygraph(self): + return self._run(to_static=False) + + def _run(self, to_static): with fluid.dygraph.guard(self.place): - ret = self.dyfunc(self.len) + if to_static: + ret = declarative(self.dyfunc)(self.len) + else: + ret = self.dyfunc(self.len) return ret.numpy() def test_ast_to_func(self): - static_numpy = self._run_static() - self._run_dygraph() self.assertTrue(np.allclose(self._run_dygraph(), self._run_static())) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py index cc1a98853e..2880dd0055 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py @@ -21,9 +21,14 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph.base import switch_to_static_graph +from paddle.fluid.dygraph import to_variable from paddle.fluid.dygraph.nn import Conv2D, Linear, Pool2D from paddle.fluid.optimizer import AdamOptimizer +from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator + +SEED = 2020 class SimpleImgConvPool(fluid.dygraph.Layer): @@ -94,7 +99,7 @@ class MNIST(fluid.dygraph.Layer): loc=0.0, scale=scale)), act="softmax") - @dygraph_to_static_func + @declarative def forward(self, inputs, label=None): x = self.inference(inputs) if label is not None: @@ -125,62 +130,96 @@ class TestMNIST(unittest.TestCase): drop_last=True) -class TestMNISTWithStaticMode(TestMNIST): +class TestMNISTWithDeclarative(TestMNIST): """ - Tests model when using `dygraph_to_static_func` to convert dygraph into static - model. It allows user to add customized code to train static model, such as `with` - and `Executor` statement. + Tests model if doesn't change the layers while decorated + by `dygraph_to_static_output`. In this case, everything should + still works if model is trained in dygraph mode. """ - def test_train(self): + def train_static(self): + return self.train(to_static=True) + + def train_dygraph(self): + return self.train(to_static=False) + + def test_mnist_declarative(self): + dygraph_loss = self.train_dygraph() + static_loss = self.train_static() + self.assertTrue( + np.allclose(dygraph_loss, static_loss), + msg='dygraph is {}\n static_res is \n{}'.format(dygraph_loss, + static_loss)) - main_prog = fluid.Program() - with fluid.program_guard(main_prog): + def train(self, to_static=False): + prog_trans = ProgramTranslator() + prog_trans.enable(to_static) + + loss_data = [] + with fluid.dygraph.guard(self.place): + fluid.default_main_program().random_seed = SEED + fluid.default_startup_program().random_seed = SEED mnist = MNIST() adam = AdamOptimizer( learning_rate=0.001, parameter_list=mnist.parameters()) - exe = fluid.Executor(self.place) - start = time() - - img = fluid.data( - name='img', shape=[None, 1, 28, 28], dtype='float32') - label = fluid.data(name='label', shape=[None, 1], dtype='int64') - label.stop_gradient = True - - prediction, acc, avg_loss = mnist(img, label) - adam.minimize(avg_loss) - exe.run(fluid.default_startup_program()) - - for epoch in range(self.epoch_num): - for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array( - [x[1] for x in data]).astype('int64').reshape(-1, 1) - - out = exe.run(main_prog, - fetch_list=[avg_loss, acc], - feed={'img': dy_x_data, - 'label': y_data}) - if batch_id % 100 == 0: - print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, - np.array(out[0]), - np.array(out[1]), time() - start)) - if batch_id == 300: - # The accuracy of mnist should converge over 0.9 after 300 batch. - accuracy = np.array(out[1]) - self.assertGreater( - accuracy, - 0.9, - msg="The accuracy {} of mnist should converge over 0.9 after 300 batch." - .format(accuracy)) + for epoch in range(self.epoch_num): + start = time() + for batch_id, data in enumerate(self.train_reader()): + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) + for x in data]).astype('float32') + y_data = np.array( + [x[1] for x in data]).astype('int64').reshape(-1, 1) + + img = to_variable(dy_x_data) + label = to_variable(y_data) + + label.stop_gradient = True + prediction, acc, avg_loss = mnist(img, label=label) + avg_loss.backward() + + adam.minimize(avg_loss) + loss_data.append(avg_loss.numpy()[0]) + # save checkpoint + mnist.clear_gradients() + if batch_id % 10 == 0: + print( + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" + .format(epoch, batch_id, + avg_loss.numpy(), + acc.numpy(), time() - start)) + start = time() + if batch_id == 50: + mnist.eval() + prediction, acc, avg_loss = mnist(img, label) + loss_data.append(avg_loss.numpy()[0]) + self.check_save_inference_model([dy_x_data, y_data], + prog_trans, to_static, + prediction) break + return loss_data + + @switch_to_static_graph + def check_save_inference_model(self, inputs, prog_trans, to_static, gt_out): + if to_static: + infer_model_path = "./test_mnist_inference_model" + prog_trans.save_inference_model(infer_model_path) + infer_out = self.load_and_run_inference(infer_model_path, inputs) + self.assertTrue(np.allclose(gt_out.numpy(), infer_out)) + + def load_and_run_inference(self, model_path, inputs): + exe = fluid.Executor(self.place) + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + dirname=model_path, executor=exe) + assert len(inputs) == len(feed_target_names) + results = exe.run(inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets) + + return np.array(results[0]) -# TODO: TestCase with cached program is required when building program in `for` loop. - if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index 266c3f3b1c..373a5e867c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -31,22 +31,24 @@ from ifelse_simple_func import dyfunc_with_if_else np.random.seed(0) +# TODO(Aurelius): Currently, `declarative` don't support decorate the function +# that contains layers with initialized operation, like `fc = linear(10, 3)`. +# Because initialized ops will be added into program and be executed many times. +# The parameters are assumed to initialized outside of the function. def simple_func(x, weight_numpy): - weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy) - linear = Linear(32, 64, param_attr=weight_initalizer, bias_attr=False) x = fluid.dygraph.to_variable(x) - y = linear(x) - z = linear(x) + w = fluid.dygraph.to_variable(weight_numpy) + y = fluid.layers.matmul(x, w) + z = fluid.layers.mean(y) return z @declarative def decorated_simple_func(x, weight_numpy): - weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy) - linear = Linear(32, 64, param_attr=weight_initalizer, bias_attr=False) x = fluid.dygraph.to_variable(x) - y = linear(x) - z = linear(x) + w = fluid.dygraph.to_variable(weight_numpy) + y = fluid.layers.matmul(x, w) + z = fluid.layers.mean(y) return z @@ -100,6 +102,14 @@ class StaticCode2(): return x_v +class NetWithError(fluid.dygraph.layers.Layer): + @declarative + def forward(self, x): + linear = fluid.dygraph.Linear(32, 64) + y = linear(x) + return y + + class TestDygraphToStaticCode(unittest.TestCase): def setUp(self): # set to print all string diff when assertEqual fails @@ -120,75 +130,79 @@ class TestDygraphToStaticCode(unittest.TestCase): class TestEnableDeclarative(unittest.TestCase): - def test_enable_disable_get_output(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() + def setUp(self): + self.x = np.random.randn(30, 10, 32).astype('float32') + self.weight = np.random.randn(32, 64).astype('float32') + self.program_translator = ProgramTranslator() - with fluid.program_guard(fluid.Program(), fluid.Program()): - program_translator.enable(True) - static_output = program_translator.get_output(simple_func, x, - weight) + def test_raise_error(self): + with fluid.dygraph.guard(): + self.program_translator.enable(True) + net = NetWithError() + with self.assertRaises(ValueError): + net(fluid.dygraph.to_variable(self.x)) - program_translator.enable(False) + def test_enable_disable_get_output(self): + self.program_translator.enable(True) with fluid.dygraph.guard(): - dygraph_output = program_translator.get_output(simple_func, x, - weight) + static_output = self.program_translator.get_output( + simple_func, self.x, self.weight) + + self.program_translator.enable(False) + with fluid.dygraph.guard(): + dygraph_output = self.program_translator.get_output( + simple_func, self.x, self.weight) self.assertTrue( np.allclose( static_output.numpy(), dygraph_output.numpy(), atol=1e-4)) def test_enable_disable_get_func(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() - with fluid.program_guard(fluid.Program(), fluid.Program()): - program_translator.enable(True) - static_func = program_translator.get_func(simple_func) + self.program_translator.enable(True) + with fluid.dygraph.guard(): + static_func = self.program_translator.get_func(simple_func) self.assertTrue(callable(static_func)) - static_output = static_func(x, weight) + static_output = static_func(self.x, self.weight) self.assertTrue(isinstance(static_output, fluid.Variable)) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_func = program_translator.get_func(simple_func) + dygraph_func = self.program_translator.get_func(simple_func) self.assertTrue(callable(dygraph_func)) - dygraph_output = dygraph_func(x, weight) + dygraph_output = dygraph_func(self.x, self.weight) self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase)) def test_enable_disable_get_program(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() - with fluid.program_guard(fluid.Program(), fluid.Program()): - program_translator.enable(True) - static_output = program_translator.get_program(simple_func, x, - weight) - self.assertTrue(isinstance(static_output, tuple)) - self.assertEqual(len(static_output), 4) - self.assertTrue(isinstance(static_output[0], fluid.Program)) - self.assertTrue(isinstance(static_output[1], fluid.Program)) - - program_translator.enable(False) + self.program_translator.enable(True) + static_output = self.program_translator.get_program(simple_func, self.x, + self.weight) + self.assertTrue(isinstance(static_output, tuple)) + self.assertEqual(len(static_output), 4) + self.assertTrue(isinstance(static_output[0], fluid.Program)) + self.assertTrue(isinstance(static_output[1], fluid.Program)) + # Check all inputs and outputs are Variable + for var in static_output[2]: + self.assertTrue(isinstance(var, fluid.Variable)) + + for var in static_output[3]: + self.assertTrue(isinstance(var, fluid.Variable)) + + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_output = program_translator.get_program(simple_func, x, - weight) + dygraph_output = self.program_translator.get_program( + simple_func, self.x, self.weight) self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase)) def test_enable_disable_declarative(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() - with fluid.program_guard(fluid.Program(), fluid.Program()): - program_translator.enable(True) - static_output = decorated_simple_func(x, weight) + self.program_translator.enable(True) + with fluid.dygraph.guard(): + static_output = decorated_simple_func(self.x, self.weight) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_output = decorated_simple_func(x, weight) + dygraph_output = decorated_simple_func(self.x, self.weight) self.assertTrue( np.allclose( static_output.numpy(), dygraph_output.numpy(), atol=1e-4)) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py new file mode 100644 index 0000000000..790319936a --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py @@ -0,0 +1,317 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +import logging +import time +import unittest + +import numpy as np + +import paddle.fluid as fluid +from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator +from paddle.fluid.dygraph.base import to_variable +from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.nn import Embedding +from paddle.fluid.optimizer import SGDOptimizer + +PRINT_STEP = 20 +SEED = 2020 + +program_translator = ProgramTranslator() + + +class SimpleLSTMRNN(fluid.Layer): + def __init__(self, + hidden_size, + num_steps, + num_layers=2, + init_scale=0.1, + dropout=None): + super(SimpleLSTMRNN, self).__init__() + self._hidden_size = hidden_size + self._num_layers = num_layers + self._init_scale = init_scale + self._dropout = dropout + self._num_steps = num_steps + self.cell_array = [] + self.hidden_array = [] + + self.weight_1_arr = [] + self.weight_2_arr = [] + self.bias_arr = [] + self.mask_array = [] + + for i in range(self._num_layers): + weight_1 = self.create_parameter( + attr=fluid.ParamAttr( + initializer=fluid.initializer.UniformInitializer( + low=-self._init_scale, high=self._init_scale)), + shape=[self._hidden_size * 2, self._hidden_size * 4], + dtype="float32", + default_initializer=fluid.initializer.UniformInitializer( + low=-self._init_scale, high=self._init_scale)) + self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) + bias_1 = self.create_parameter( + attr=fluid.ParamAttr( + initializer=fluid.initializer.UniformInitializer( + low=-self._init_scale, high=self._init_scale)), + shape=[self._hidden_size * 4], + dtype="float32", + default_initializer=fluid.initializer.Constant(0.0)) + self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) + + def forward(self, input_embedding, init_hidden=None, init_cell=None): + cell_array = [] + hidden_array = [] + + for i in range(self._num_layers): + hidden_array.append(init_hidden[i]) + cell_array.append(init_cell[i]) + + res = [] + for index in range(self._num_steps): + step_input = input_embedding[:, index, :] + for k in range(self._num_layers): + pre_hidden = hidden_array[k] + pre_cell = cell_array[k] + weight_1 = self.weight_1_arr[k] + bias = self.bias_arr[k] + + nn = fluid.layers.concat([step_input, pre_hidden], 1) + gate_input = fluid.layers.matmul(x=nn, y=weight_1) + + gate_input = fluid.layers.elementwise_add(gate_input, bias) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1) + c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( + i) * fluid.layers.tanh(j) + m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) + hidden_array[k] = m + cell_array[k] = c + step_input = m + + if self._dropout is not None and self._dropout > 0.0: + step_input = fluid.layers.dropout( + step_input, + dropout_prob=self._dropout, + dropout_implementation='upscale_in_train') + res.append(step_input) + real_res = fluid.layers.concat(res, 1) + real_res = fluid.layers.reshape( + real_res, [-1, self._num_steps, self._hidden_size]) + last_hidden = fluid.layers.concat(hidden_array, 1) + last_hidden = fluid.layers.reshape( + last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) + last_cell = fluid.layers.concat(cell_array, 1) + last_cell = fluid.layers.reshape( + last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) + return real_res, last_hidden, last_cell + + +class PtbModel(fluid.Layer): + def __init__(self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None): + super(PtbModel, self).__init__() + self.hidden_size = hidden_size + self.vocab_size = vocab_size + self.init_scale = init_scale + self.num_layers = num_layers + self.num_steps = num_steps + self.dropout = dropout + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout) + self.embedding = Embedding( + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False, + param_attr=fluid.ParamAttr( + name='embedding_para', + initializer=fluid.initializer.UniformInitializer( + low=-init_scale, high=init_scale))) + self.softmax_weight = self.create_parameter( + attr=fluid.ParamAttr(), + shape=[self.hidden_size, self.vocab_size], + dtype="float32", + default_initializer=fluid.initializer.UniformInitializer( + low=-self.init_scale, high=self.init_scale)) + self.softmax_bias = self.create_parameter( + attr=fluid.ParamAttr(), + shape=[self.vocab_size], + dtype="float32", + default_initializer=fluid.initializer.UniformInitializer( + low=-self.init_scale, high=self.init_scale)) + + def build_once(self, input, label, init_hidden, init_cell): + pass + + @declarative + def forward(self, input, label, init_hidden, init_cell): + + init_h = fluid.layers.reshape( + init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + + init_c = fluid.layers.reshape( + init_cell, shape=[self.num_layers, -1, self.hidden_size]) + + x_emb = self.embedding(input) + + x_emb = fluid.layers.reshape( + x_emb, shape=[-1, self.num_steps, self.hidden_size]) + if self.dropout is not None and self.dropout > 0.0: + x_emb = fluid.layers.dropout( + x_emb, + dropout_prob=self.dropout, + dropout_implementation='upscale_in_train') + rnn_out, last_hidden, last_cell = self.simple_lstm_rnn(x_emb, init_h, + init_c) + + projection = fluid.layers.matmul(rnn_out, self.softmax_weight) + projection = fluid.layers.elementwise_add(projection, self.softmax_bias) + + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False) + loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) + loss = fluid.layers.reduce_mean(loss, dim=[0]) + loss = fluid.layers.reduce_sum(loss) + + return loss, last_hidden, last_cell + + def debug_emb(self): + + np.save("emb_grad", self.x_emb.gradient()) + + +def train(place): + + num_layers = 1 + batch_size = 4 + hidden_size = 10 + num_steps = 3 + init_scale = 0.1 + max_epoch = 1 + dropout = 0.0 + vocab_size = 1000 + batch_num = 200 + + with fluid.dygraph.guard(place): + fluid.default_startup_program().random_seed = SEED + fluid.default_main_program().random_seed = SEED + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters()) + + for epoch_id in range(max_epoch): + + total_loss = 0.0 + iters = 0.0 + total_sample = 0 + + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + + init_hidden = to_variable(init_hidden_data) + init_cell = to_variable(init_cell_data) + for step_id in range(batch_num): + x_data = np.arange(12).reshape(4, 3).astype('int64') + y_data = np.arange(1, 13).reshape(4, 3).astype('int64') + y_data = y_data.reshape((-1, 1)) + + x_data = x_data.reshape((-1, num_steps, 1)) + y_data = y_data.reshape((-1, num_steps, 1)) + + x = to_variable(x_data) + y = to_variable(y_data) + + dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden, + init_cell) + out_loss = dy_loss.numpy() + + dy_loss.backward() + sgd.minimize(dy_loss) + ptb_model.clear_gradients() + + total_loss += out_loss + iters += num_steps + total_sample += 1 + if step_id % PRINT_STEP == 0: + if step_id == 0: + logging.info("epoch %d | step %d, loss %0.3f" % ( + epoch_id, step_id, total_loss / total_sample)) + avg_batch_time = time.time() + else: + speed = PRINT_STEP / (time.time() - avg_batch_time) + logging.info( + "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" + % (epoch_id, step_id, total_loss / total_sample, + speed)) + avg_batch_time = time.time() + + return out_loss, last_hidden.numpy(), last_cell.numpy() + + +def train_dygraph(place): + program_translator.enable(False) + return train(place) + + +def train_static(place): + program_translator.enable(True) + return train(place) + + +class TestPtb(unittest.TestCase): + def setUp(self): + self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + else fluid.CPUPlace() + + def test_check_result(self): + loss_1, hidden_1, cell_1 = train_static(self.place) + loss_2, hidden_2, cell_2 = train_dygraph(self.place) + + self.assertTrue( + np.allclose(loss_1, loss_2), + msg="static loss: {} \ndygraph loss: {}".format(loss_1, loss_2)) + self.assertTrue( + np.allclose(hidden_1, hidden_2), + msg="static hidden: {} \ndygraph acc1: {}".format(hidden_1, + hidden_2)) + self.assertTrue( + np.allclose(cell_1, cell_2), + msg="static cell: {} \ndygraph cell: {}".format(cell_1, cell_2)) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_recursive_call.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_recursive_call.py index fe6610ec0c..18645efad2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_recursive_call.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_recursive_call.py @@ -19,14 +19,17 @@ import unittest import numpy as np import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph import ProgramTranslator +from paddle.fluid.dygraph import declarative + +program_translator = ProgramTranslator() SEED = 2020 np.random.seed(SEED) # Use a decorator to test exception -@dygraph_to_static_func +@declarative def dyfunc_with_if(x_v): if fluid.layers.mean(x_v).numpy()[0] > 5: x_v = x_v - 1 @@ -35,7 +38,7 @@ def dyfunc_with_if(x_v): return x_v -@dygraph_to_static_func +@declarative def nested_func(x_v): x_v = fluid.dygraph.to_variable(x_v) @@ -57,17 +60,16 @@ class TestRecursiveCall1(unittest.TestCase): self.dyfunc = nested_func def get_dygraph_output(self): + program_translator.enable(False) with fluid.dygraph.guard(): res = self.dyfunc(self.input).numpy() return res def get_static_output(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - static_out = self.dyfunc(self.input) - exe = fluid.Executor(self.place) - static_res = exe.run(main_program, fetch_list=static_out) - return static_res[0] + program_translator.enable(True) + with fluid.dygraph.guard(): + res = self.dyfunc(self.input).numpy() + return res def test_transformed_static_result(self): static_res = self.get_static_output() @@ -93,14 +95,14 @@ class MyConvLayer(fluid.dygraph.Layer): bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.5))) - @dygraph_to_static_func + @declarative def forward(self, inputs): y = dyfunc_with_if(inputs) y = lambda_fun(y) y = self.dymethod(y) return y - @dygraph_to_static_func + @declarative def dymethod(self, x_v): x_v = fluid.layers.assign(x_v) return x_v @@ -120,7 +122,7 @@ class MyLayer(fluid.dygraph.Layer): bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.5))) - @dygraph_to_static_func + @declarative def forward(self, inputs): h = self.conv(inputs) out = self.fc(h) @@ -134,7 +136,7 @@ class TestRecursiveCall2(unittest.TestCase): self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda( ) else fluid.CPUPlace() - def get_dygraph_output(self): + def _run(self): with fluid.dygraph.guard(): self.dygraph_func = self.Layer() fluid.default_startup_program.random_seed = SEED @@ -144,21 +146,13 @@ class TestRecursiveCall2(unittest.TestCase): return res.numpy() - def get_static_output(self): - startup_program = fluid.Program() - startup_program.random_seed = SEED - main_program = fluid.Program() - main_program.random_seed = SEED - - with fluid.program_guard(main_program, startup_program): - self.dygraph_func = self.Layer() - data = fluid.layers.assign(self.input) - static_out = self.dygraph_func(data) + def get_dygraph_output(self): + program_translator.enable(False) + return self._run() - exe = fluid.Executor(self.place) - exe.run(startup_program) - static_res = exe.run(main_program, fetch_list=static_out) - return static_res[0] + def get_static_output(self): + program_translator.enable(True) + return self._run() def test_transformed_static_result(self): dygraph_res = self.get_dygraph_output() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py index af223c2b00..7414d240bf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py @@ -22,8 +22,11 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from -np.random.seed(2020) +SEED = 2020 + +np.random.seed(SEED) place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace( ) @@ -36,7 +39,6 @@ class SimpleFcLayer(fluid.dygraph.Layer): @declarative def forward(self, x): - x = fluid.dygraph.to_variable(x) y = self._linear(x) z = self._linear(y) out = fluid.layers.mean(z) @@ -46,37 +48,90 @@ class SimpleFcLayer(fluid.dygraph.Layer): class TestDyToStaticSaveInferenceModel(unittest.TestCase): def test_save_inference_model(self): fc_size = 20 - - x = np.random.random((fc_size, fc_size)).astype('float32') - layer = SimpleFcLayer(fc_size) - - program_translator = ProgramTranslator.get_instance() - adam = fluid.optimizer.SGD(learning_rate=0.001) - program_translator.set_optimizer(adam, index_of_loss=0) - - for i in range(5): - out = layer(x) - - main_program = ProgramTranslator.get_instance().main_program - expected_persistable_vars = set( - [layer._linear.weight.name, layer._linear.bias.name]) + x_data = np.random.random((fc_size, fc_size)).astype('float32') + with fluid.dygraph.guard(place): + fluid.default_startup_program().random_seed = SEED + fluid.default_main_program().random_seed = SEED + + x = fluid.dygraph.to_variable(x_data) + layer = SimpleFcLayer(fc_size) + adam = fluid.optimizer.SGD(learning_rate=0.1, + parameter_list=layer.parameters()) + + for i in range(5): + loss, _ = layer(x) + loss.backward() + adam.minimize(loss) + layer.clear_gradients() + # Check the correctness of the inference + dygraph_out, _ = layer(x) + self.check_save_inference_model(layer, [x_data], dygraph_out.numpy()) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), fetch=[0]) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), feed=[0]) + + def check_save_inference_model(self, + model, + inputs, + gt_out, + feed=None, + fetch=None): + program_translator = ProgramTranslator() + expected_persistable_vars = set([p.name for p in model.parameters()]) infer_model_dir = "./test_dy2stat_save_inference_model" - ProgramTranslator.get_instance().save_inference_model(infer_model_dir) - saved_var_names = set([ - filename for filename in os.listdir(infer_model_dir) - if filename != '__model__' - ]) - self.assertEqual(saved_var_names, expected_persistable_vars) - - infer_model_dir = "./test_dy2stat_save_inference_model_with_fetch" - ProgramTranslator.get_instance().save_inference_model( - infer_model_dir, fetch=[0]) + program_translator.save_inference_model( + infer_model_dir, feed=feed, fetch=fetch) saved_var_names = set([ filename for filename in os.listdir(infer_model_dir) if filename != '__model__' ]) self.assertEqual(saved_var_names, expected_persistable_vars) + # Check the correctness of the inference + infer_out = self.load_and_run_inference(infer_model_dir, inputs) + self.assertTrue(np.allclose(gt_out, infer_out)) + + def load_and_run_inference(self, model_path, inputs): + exe = fluid.Executor(place) + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + dirname=model_path, executor=exe) + results = exe.run(inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets) + + return np.array(results[0]) + + +class TestPartialProgramRaiseError(unittest.TestCase): + def test_param_type(self): + program_translator = ProgramTranslator() + program_translator.enable(True) + x_data = np.random.random((20, 20)).astype('float32') + + with fluid.dygraph.guard(fluid.CPUPlace()): + net = SimpleFcLayer(20) + x = fluid.dygraph.to_variable(x_data) + out = net(x) + + program_cache = program_translator.get_program_cache() + _, (concrete_program, _) = program_cache.last() + + params = concrete_program.parameters + + concrete_program.parameters = params[0] + # TypeError: Type of self._params should be list or tuple, + # but received . + with self.assertRaises(TypeError): + partial_program_from(concrete_program) + + params[0] = "linear.w.0" + concrete_program.parameters = params + # TypeError: Type of self._params[0] should be framework.ParamBase, + # but received . + with self.assertRaises(TypeError): + partial_program_from(concrete_program) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py index 36467e2f6e..54eefe7c4f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py @@ -18,10 +18,10 @@ import unittest import numpy as np import paddle.fluid as fluid -import paddle.fluid.framework as framework from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator -from paddle.fluid.dygraph.nn import Linear +from paddle.fluid.optimizer import AdamOptimizer +from test_fetch_feed import Linear np.random.seed(2020) @@ -29,53 +29,50 @@ place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace( ) -def simple_func(x, weight_numpy): - weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy) - linear = Linear(32, 64, param_attr=weight_initalizer) - x = fluid.dygraph.to_variable(x) - y = linear(x) - z = linear(x) - return z +class TestDyToStaticSaveLoad(unittest.TestCase): + def test_save_load_same_result(self): + program_translator = ProgramTranslator() + x_data = np.random.randn(30, 10, 32).astype('float32') + batch_num = 3 + + with fluid.dygraph.guard(place): + program_translator.enable(True) + x = fluid.dygraph.to_variable(x_data) + net = Linear(32, 64) + adam = AdamOptimizer( + learning_rate=0.1, parameter_list=net.parameters()) + + for i in range(batch_num): + static_out, static_loss = net(x) + # Update parameters + static_loss.backward() + adam.minimize(static_loss) + net.clear_gradients() + # Save parameters + fluid.save_dygraph(net.state_dict(), "./test_dy2stat_save_load") + # minimize() will update parameter, call net() to get output and avg_loss. + # Switch into eval mode. + net.eval() + static_out, static_loss = net(x) + + # load parameters into dygraph + with fluid.dygraph.guard(place): + dygraph_net = Linear(32, 64) -def decorated_simple_func(x, weight_numpy): - weight_initalizer = fluid.initializer.NumpyArrayInitializer(weight_numpy) - linear = Linear(32, 64, param_attr=weight_initalizer) - x = fluid.dygraph.to_variable(x) - y = linear(x) - z = linear(x) - return z + # Load parameters + model_dict, _ = fluid.load_dygraph("./test_dy2stat_save_load") + dygraph_net.set_dict(model_dict) + # Switch into eval mode. + dygraph_net.eval() + x = fluid.dygraph.to_variable(x_data) + # predict output + program_translator.enable(False) + dygraph_out, dygraph_loss = dygraph_net(x) -class TestDyToStaticSaveLoad(unittest.TestCase): - def test_save_load_same_result(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - with fluid.dygraph.guard(place): - dygraph_result = simple_func(x, weight) - - main_program, startup_program, inputs, outputs = ProgramTranslator( - ).get_program(decorated_simple_func, x, weight) - exe = fluid.Executor(place) - exe.run(startup_program) - fluid.save(main_program, "./test_dy2stat_save_load") - - # set vars to zero so that we can test load in same file - for var in main_program.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - tensor = fluid.global_scope().find_var(var.name).get_tensor() - tensor.set(np.zeros_like(np.array(tensor)), place) - - # make sure all the paramerter or optimizer var have been set to zero - tensor_np = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - self.assertEqual(0, np.sum(np.abs(tensor_np))) - - fluid.load(main_program, "./test_dy2stat_save_load") - static_result = exe.run(main_program, - feed={inputs[0].name: x}, - fetch_list=outputs) - self.assertTrue(np.allclose(dygraph_result.numpy(), static_result)) + self.assertTrue(np.allclose(dygraph_out.numpy(), static_out.numpy())) + self.assertTrue(np.allclose(dygraph_loss.numpy(), static_loss.numpy())) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py index 0bdbaed436..acef2f9d8d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py @@ -22,8 +22,9 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.base import to_variable -from paddle.fluid.dygraph.jit import dygraph_to_static_func from paddle.fluid.dygraph.nn import BatchNorm, Conv2D, Linear, Pool2D +from paddle.fluid.dygraph import declarative +from paddle.fluid.dygraph import ProgramTranslator SEED = 2020 np.random.seed(SEED) @@ -286,7 +287,7 @@ class SeResNeXt(fluid.dygraph.Layer): param_attr=fluid.param_attr.ParamAttr( initializer=fluid.initializer.Uniform(-stdv, stdv))) - @dygraph_to_static_func + @declarative def forward(self, inputs, label): if self.layers == 50 or self.layers == 101: y = self.conv0(inputs) @@ -314,7 +315,10 @@ class SeResNeXt(fluid.dygraph.Layer): return out, avg_loss, acc_top1, acc_top5 -def train_dygraph(train_reader): +def train(train_reader, to_static): + program_translator = ProgramTranslator() + program_translator.enable(to_static) + np.random.seed(SEED) with fluid.dygraph.guard(place): @@ -374,75 +378,6 @@ def train_dygraph(train_reader): ) -def train_static(train_reader): - np.random.seed(SEED) - exe = fluid.Executor(place) - - main_prog = fluid.Program() - startup_prog = fluid.Program() - with fluid.program_guard(main_prog, startup_prog): - with fluid.unique_name.guard(): - main_prog.random_seed = SEED - startup_prog.random_seed = SEED - img = fluid.data( - name="img", shape=[None, 3, 224, 224], dtype="float32") - label = fluid.data(name="label", shape=[None, 1], dtype="int64") - label.stop_gradient = True - - se_resnext = SeResNeXt() - pred, avg_loss_, acc_top1_, acc_top5_ = se_resnext(img, label) - - optimizer = optimizer_setting(train_parameters, - se_resnext.parameters()) - optimizer.minimize(avg_loss_) - - exe.run(startup_prog) - - for epoch_id in range(EPOCH_NUM): - total_loss = 0.0 - total_acc1 = 0.0 - total_acc5 = 0.0 - total_sample = 0 - step_idx = 0 - speed_list = [] - for step_id, data in enumerate(train_reader()): - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data]).astype('int64').reshape( - BATCH_SIZE, 1) - - pred_, avg_loss, acc_top1, acc_top5 = exe.run( - main_prog, - feed={"img": dy_x_data, - "label": y_data}, - fetch_list=[pred, avg_loss_, acc_top1_, acc_top5_]) - - total_loss += avg_loss - total_acc1 += acc_top1 - total_acc5 += acc_top5 - total_sample += 1 - - if step_id % PRINT_STEP == 0: - if step_id == 0: - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample)) - avg_batch_time = time.time() - else: - speed = PRINT_STEP / (time.time() - avg_batch_time) - speed_list.append(speed) - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample, speed)) - avg_batch_time = time.time() - - step_idx += 1 - if step_idx == STEP_NUM: - break - - return pred_, avg_loss, acc_top1, acc_top5 - - class TestSeResnet(unittest.TestCase): def setUp(self): self.train_reader = paddle.batch( @@ -452,8 +387,10 @@ class TestSeResnet(unittest.TestCase): drop_last=True) def test_check_result(self): - pred_1, loss_1, acc1_1, acc5_1 = train_static(self.train_reader) - pred_2, loss_2, acc1_2, acc5_2 = train_dygraph(self.train_reader) + pred_1, loss_1, acc1_1, acc5_1 = train( + self.train_reader, to_static=False) + pred_2, loss_2, acc1_2, acc5_2 = train( + self.train_reader, to_static=True) self.assertTrue( np.allclose(pred_1, pred_2), diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py index 27d49404ce..3a450d4554 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py @@ -69,6 +69,11 @@ def test_slice_in_while_loop(x, iter_num): def test_slice_in_for_loop(x, iter_num): x = fluid.dygraph.to_variable(x) a = [] + # Use `fill_constant` so that static analysis can analyze the type of iter_num is Tensor + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) # TODO(liym27): Delete it if the type of parameter iter_num can be resolved + for i in range(iter_num): a.append(x) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py index 39b8d450b8..6acc4bcd1c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py @@ -18,7 +18,7 @@ import numpy import unittest import paddle.fluid as fluid -from paddle.fluid.dygraph.jit import dygraph_to_static_func +from paddle.fluid.dygraph.jit import declarative def dyfunc_tensor_shape_1(x): @@ -171,20 +171,19 @@ class TestTensorShapeBasic(unittest.TestCase): def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_1 - def get_dygraph_output(self): + def _run(self, to_static): with fluid.dygraph.guard(): - res = self.dygraph_func(self.input).numpy() + if to_static: + res = declarative(self.dygraph_func)(self.input).numpy() + else: + res = self.dygraph_func(self.input).numpy() return res - def get_static_output(self): - main_program = fluid.Program() - with fluid.program_guard(main_program): - static_out = dygraph_to_static_func(self.dygraph_func)(self.input) - - exe = fluid.Executor(self.place) - static_res = exe.run(main_program, fetch_list=static_out) + def get_dygraph_output(self): + return self._run(to_static=False) - return static_res[0] + def get_static_output(self): + return self._run(to_static=False) def test_transformed_static_result(self): static_res = self.get_static_output() -- GitLab