From ab85f87aa9328b6ba6629ef9a983510b27cc2dfd Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 1 Dec 2022 21:40:42 +0800 Subject: [PATCH] [Fluid Clean]Migrate utils files and delete dygraph_to_static dir (#48566) * [Fluid Clean]Migrate utils files and delete dygraph_to_static dir * fix setup.py.in * fix import * fix unittest * fix code style * fix unittest --- python/paddle/fluid/compiler.py | 2 +- python/paddle/fluid/dygraph/__init__.py | 2 - .../dygraph/dygraph_to_static/__init__.py | 28 ------ .../dygraph_to_static/variable_trans_func.py | 95 ------------------- python/paddle/fluid/layers/control_flow.py | 10 +- .../dygraph_to_static/test_ast_util.py | 2 +- .../test_basic_api_transformation.py | 2 +- .../dygraph_to_static/test_break_continue.py | 2 +- .../test_closure_analysis.py | 4 +- .../unittests/dygraph_to_static/test_error.py | 4 +- .../dygraph_to_static/test_function_spec.py | 2 +- .../dygraph_to_static/test_ifelse.py | 2 +- .../dygraph_to_static/test_logging_utils.py | 2 +- .../dygraph_to_static/test_origin_info.py | 8 +- .../test_program_translator.py | 2 +- .../dygraph_to_static/test_return.py | 2 +- .../dygraph_to_static/test_rollback.py | 2 +- .../dygraph_to_static/test_setter_helper.py | 2 +- .../dygraph_to_static/test_static_analysis.py | 5 +- .../unittests/dygraph_to_static/test_utils.py | 5 +- .../test_variable_trans_func.py | 6 +- .../paddle/fluid/tests/unittests/op_test.py | 2 +- .../fluid/tests/unittests/test_input_spec.py | 4 +- python/paddle/fluid/variable_index.py | 2 +- python/paddle/jit/api.py | 8 +- python/paddle/jit/dy2static/__init__.py | 7 +- .../jit/dy2static/assert_transformer.py | 4 +- .../paddle/jit/dy2static/ast_transformer.py | 6 +- python/paddle/jit/dy2static/base.py | 18 ---- .../paddle/jit/dy2static/base_transformer.py | 2 +- .../jit/dy2static/basic_api_transformer.py | 7 +- .../dy2static/break_continue_transformer.py | 6 +- .../paddle/jit/dy2static/call_transformer.py | 6 +- .../paddle/jit/dy2static/cast_transformer.py | 4 +- .../paddle/jit/dy2static/convert_call_func.py | 4 +- .../paddle/jit/dy2static/convert_operators.py | 6 +- .../dy2static/create_variable_transformer.py | 6 +- .../jit/dy2static/decorator_transformer.py | 4 +- .../jit/dy2static/early_return_transformer.py | 2 +- .../dy2static}/error.py | 13 +-- .../dy2static}/function_spec.py | 13 ++- .../jit/dy2static/ifelse_transformer.py | 14 +-- .../dy2static}/logging_utils.py | 2 +- .../jit/dy2static/logical_transformer.py | 2 +- .../paddle/jit/dy2static/loop_transformer.py | 16 ++-- .../dy2static}/origin_info.py | 7 +- .../paddle/jit/dy2static/partial_program.py | 2 +- .../paddle/jit/dy2static/print_transformer.py | 2 +- .../jit/dy2static/program_translator.py | 30 +++--- .../jit/dy2static/return_transformer.py | 8 +- .../dy2static}/static_analysis.py | 0 .../jit/dy2static/tensor_shape_transformer.py | 4 +- .../jit/dy2static/typehint_transformer.py | 2 +- .../dy2static}/utils.py | 36 +++---- .../jit/dy2static/variable_trans_func.py | 81 +++++++++++++++- python/setup.py.in | 1 - 56 files changed, 217 insertions(+), 303 deletions(-) delete mode 100644 python/paddle/fluid/dygraph/dygraph_to_static/__init__.py delete mode 100644 python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py delete mode 100644 python/paddle/jit/dy2static/base.py rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/error.py (98%) rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/function_spec.py (97%) rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/logging_utils.py (99%) rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/origin_info.py (98%) rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/static_analysis.py (100%) rename python/paddle/{fluid/dygraph/dygraph_to_static => jit/dy2static}/utils.py (98%) diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index 81af46c468..f763e0f1d8 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -696,7 +696,7 @@ class IpuDynamicPatcher: ProgramCache, MAX_TRACED_PROGRAM_COUNT, ) - from ..fluid.dygraph.dygraph_to_static import logging_utils + from paddle.jit.dy2static import logging_utils from paddle.jit.dy2static.partial_program import ( partial_program_from, ) diff --git a/python/paddle/fluid/dygraph/__init__.py b/python/paddle/fluid/dygraph/__init__.py index 76c75b974e..551561428d 100644 --- a/python/paddle/fluid/dygraph/__init__.py +++ b/python/paddle/fluid/dygraph/__init__.py @@ -43,8 +43,6 @@ from .io import * from . import static_runner from .static_runner import StaticModelRunner -from . import dygraph_to_static - from . import rnn from .rnn import * diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py b/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py deleted file mode 100644 index 15013fb36d..0000000000 --- a/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import static_analysis -from .static_analysis import * - -from . import variable_trans_func -from .variable_trans_func import * - - -from . import logging_utils -from .logging_utils import * - -__all__ = [] -__all__ += static_analysis.__all__ -__all__ += variable_trans_func.__all__ -__all__ += logging_utils.__all__ diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py deleted file mode 100644 index f28f199362..0000000000 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle -import textwrap -from paddle.utils import gast -from paddle.fluid import unique_name -from paddle.fluid.framework import Variable -from paddle.fluid.dygraph.dygraph_to_static.utils import ( - UndefinedVar, - create_undefined_variable, -) -from paddle.fluid.layers.utils import map_structure, is_sequence - -__all__ = [ - 'create_bool_as_type', - 'create_fill_constant_node', - 'to_static_variable', - 'create_undefined_var', -] - - -def create_undefined_var(name): - func_code = "{} = _jst.UndefinedVar('{}')".format(name, name) - return gast.parse(func_code).body[0] - - -def create_fill_constant_node(name, value=0): - func_code = "{} = paddle.full(shape=[1], ".format(name) - if isinstance(value, bool): - func_code += "dtype='bool', fill_value={}, name='{}')".format( - value, name - ) - return gast.parse(func_code).body[0] - if isinstance(value, float): - func_code += "dtype='float64', fill_value={}, name='{}')".format( - value, name - ) - return gast.parse(func_code).body[0] - - if isinstance(value, int): - func_code += "dtype='int64', fill_value={}, name='{}')".format( - value, name - ) - return gast.parse(func_code).body[0] - - -def to_static_variable(x): - ''' - Translate a Python Tensor to PaddlePaddle static graph Tensor - ''' - if isinstance(x, bool): - return paddle.full(shape=[1], dtype='bool', fill_value=x) - if isinstance(x, float): - return paddle.full(shape=[1], dtype='float64', fill_value=x) - if isinstance(x, int): - return paddle.full(shape=[1], dtype='int64', fill_value=x) - if isinstance(x, UndefinedVar) or x is None: - """ - for early return case, we need a variable to represent None, current we use data_layer_not_check. - """ - return create_undefined_variable() - if is_sequence(x): - return map_structure(to_static_variable, x) - return x - - -def create_bool_as_type(x, value=True): - ''' - Create a bool variable, which type is the same as x. - ''' - if isinstance(x, Variable): - return paddle.full(shape=[1], fill_value=value, dtype="bool") - else: - return value - - -def create_bool_node(name, value): - ''' - Create a assign stmt for name = value . - ''' - assert isinstance(value, bool) - node = "{} = {}".format(name, value) - return gast.parse(node).body[0] diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index c47e9babea..7a6079ad62 100755 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -159,10 +159,10 @@ def select_input(inputs, mask): def select_input_with_buildin_type(inputs, mask, name): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + from paddle.jit.dy2static.variable_trans_func import ( to_static_variable, ) - from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar + from paddle.jit.dy2static.utils import UndefinedVar false_var, true_var = inputs @@ -1484,7 +1484,7 @@ def _deal_with_undefined_var(output_vars, loop_vars): 3. UndefinedVar = List(int) # create a list of variable 4. UndefinedVar = value # create a variable """ - from paddle.fluid.dygraph.dygraph_to_static.utils import ( + from paddle.jit.dy2static.utils import ( UndefinedVar, create_undefined_variable, ) @@ -2552,7 +2552,7 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): def change_none_to_undefinedvar(nest1, nest2): - from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar + from paddle.jit.dy2static.utils import UndefinedVar def map_fn(x): if x is None: @@ -2588,7 +2588,7 @@ def expand_undefined_var(nest1, nest2, names): nest2: Var2, ([1,2,3,4], UndefinedVar) In this case, we should not expand recursively. """ - from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar + from paddle.jit.dy2static.utils import UndefinedVar from paddle.jit.dy2static.return_transformer import ( RETURN_VALUE_PREFIX, ) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py index b81ed5f0b4..b417cd5fd9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py @@ -25,7 +25,7 @@ from ifelse_simple_func import ( import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func +from paddle.jit.dy2static.utils import ast_to_func from paddle.utils import gast diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index 34a65913c5..3733977c5d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -22,8 +22,8 @@ import paddle.fluid as fluid import paddle.fluid.dygraph as dygraph from paddle import to_tensor from paddle.fluid.dygraph import to_variable -from paddle.fluid.dygraph.dygraph_to_static.utils import is_dygraph_api from paddle.jit.api import dygraph_to_static_func +from paddle.jit.dy2static.utils import is_dygraph_api from paddle.utils import gast SEED = 2020 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py index b3d1b5b2cb..f81c2d5364 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py @@ -18,9 +18,9 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException from paddle.jit.api import declarative from paddle.jit.dy2static.program_translator import ProgramTranslator +from paddle.jit.dy2static.utils import Dygraph2StaticException SEED = 2020 np.random.seed(SEED) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py index b015669916..bed90ccbe4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py @@ -18,9 +18,7 @@ import unittest from numpy import append import paddle -from paddle.fluid.dygraph.dygraph_to_static.utils import ( - FunctionNameLivenessAnalysis, -) +from paddle.jit.dy2static.utils import FunctionNameLivenessAnalysis from paddle.utils import gast global_a = [] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py index 6faed1a61e..8059708144 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py @@ -18,8 +18,8 @@ import unittest import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.dygraph_to_static import error -from paddle.fluid.dygraph.dygraph_to_static.origin_info import unwrap +from paddle.jit.dy2static import error +from paddle.jit.dy2static.origin_info import unwrap def inner_func(): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py index b7a26169f3..ac9f6362cb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py @@ -17,7 +17,7 @@ import unittest from test_declarative import foo_func import paddle -from paddle.fluid.dygraph.dygraph_to_static.function_spec import FunctionSpec +from paddle.jit.dy2static.function_spec import FunctionSpec from paddle.static import InputSpec paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py index 89b9e87156..c17bfd2508 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py @@ -43,9 +43,9 @@ from ifelse_simple_func import ( import paddle import paddle.fluid.core as core -from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException from paddle.jit.api import declarative from paddle.jit.dy2static.program_translator import ProgramTranslator +from paddle.jit.dy2static.utils import Dygraph2StaticException np.random.seed(1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py index 93e5fae00d..fa34869e92 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py @@ -20,7 +20,7 @@ import unittest from unittest import mock import paddle -from paddle.fluid.dygraph.dygraph_to_static import logging_utils +from paddle.jit.dy2static import logging_utils from paddle.utils import gast diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py index 65c8fb4a1f..540ad3c13b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py @@ -15,7 +15,9 @@ import sys import unittest -from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( +from paddle.jit.api import declarative +from paddle.jit.dy2static import DygraphToStaticAst +from paddle.jit.dy2static.origin_info import ( ORIGI_INFO, Location, OriginInfo, @@ -25,9 +27,7 @@ from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( inspect, unwrap, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from paddle.jit.api import declarative -from paddle.jit.dy2static import DygraphToStaticAst +from paddle.jit.dy2static.utils import ast_to_func def simple_func(x): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index c61fdcccf0..aee91f6de1 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -27,9 +27,9 @@ from ifelse_simple_func import ( import paddle import paddle.fluid as fluid import paddle.jit.dy2static as _jst -from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code from paddle.jit import ProgramTranslator from paddle.jit.api import declarative +from paddle.jit.dy2static.utils import func_to_source_code from paddle.utils import gast np.random.seed(0) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py index 76e3de2a5e..b4ec4fb8fd 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py @@ -20,8 +20,8 @@ from ifelse_simple_func import dyfunc_with_if_else import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException from paddle.jit import ProgramTranslator, to_static +from paddle.jit.dy2static.utils import Dygraph2StaticException SEED = 2020 np.random.seed(SEED) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py index e7dabd9af3..c418a850d5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py @@ -17,8 +17,8 @@ import unittest import numpy as np import paddle -from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code from paddle.jit.dy2static.program_translator import StaticFunction +from paddle.jit.dy2static.utils import func_to_source_code class Net(paddle.nn.Layer): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py index 725dc032d5..eab182b6f4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py @@ -14,7 +14,7 @@ import unittest -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper +from paddle.jit.dy2static.utils import GetterSetterHelper vars = [1, 2, 3, 4, 5] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py index 22598ede71..23eb6964c3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py @@ -19,10 +19,7 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.dygraph_to_static import ( - NodeVarType, - StaticAnalysisVisitor, -) +from paddle.jit.dy2static import NodeVarType, StaticAnalysisVisitor from paddle.utils import gast diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py index a310ab6530..3361a866fe 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py @@ -15,10 +15,7 @@ import types import unittest -from paddle.fluid.dygraph.dygraph_to_static.utils import ( - index_in_list, - is_paddle_func, -) +from paddle.jit.dy2static.utils import index_in_list, is_paddle_func class TestIndexInList(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py index 353a5e8b79..f2395fa517 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py @@ -14,10 +14,8 @@ import unittest -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( - create_fill_constant_node, -) +from paddle.jit.dy2static.utils import ast_to_source_code +from paddle.jit.dy2static.variable_trans_func import create_fill_constant_node class TestVariableTransFunc(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index fd43a80c17..9728edf5d1 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -29,7 +29,6 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid import unique_name from paddle.fluid.backward import append_backward -from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs from paddle.fluid.executor import Executor from paddle.fluid.framework import ( OpProtoHolder, @@ -43,6 +42,7 @@ from paddle.fluid.framework import ( _test_eager_guard, ) from paddle.fluid.op import Operator +from paddle.jit.dy2static.utils import parse_arg_and_kwargs sys.path.append(os.path.abspath(os.path.dirname(__file__))) from testsuite import append_input_output, append_loss_ops, create_op, set_input diff --git a/python/paddle/fluid/tests/unittests/test_input_spec.py b/python/paddle/fluid/tests/unittests/test_input_spec.py index 3a623f463c..07a2e4da44 100644 --- a/python/paddle/fluid/tests/unittests/test_input_spec.py +++ b/python/paddle/fluid/tests/unittests/test_input_spec.py @@ -20,10 +20,8 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.dygraph_to_static.utils import ( - _compatible_non_tensor_spec, -) from paddle.fluid.framework import convert_np_dtype_to_dtype_ +from paddle.jit.dy2static.utils import _compatible_non_tensor_spec from paddle.static import InputSpec diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py index cf298501a2..31d587269d 100644 --- a/python/paddle/fluid/variable_index.py +++ b/python/paddle/fluid/variable_index.py @@ -622,7 +622,7 @@ def _setitem_for_tensor_array(var, item, value): not _non_static_mode() ), "setitem for tensor_array must be called in static graph mode." if isinstance(item, (Variable, int)): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + from paddle.jit.dy2static.variable_trans_func import ( to_static_variable, ) from paddle import cast diff --git a/python/paddle/jit/api.py b/python/paddle/jit/api.py index 0387460c26..95b07a989a 100644 --- a/python/paddle/jit/api.py +++ b/python/paddle/jit/api.py @@ -34,16 +34,16 @@ from paddle.fluid.dygraph.base import ( program_desc_tracing_guard, switch_to_static_graph, ) -from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.jit.dy2static.convert_call_func import ( +from .dy2static import logging_utils +from .dy2static.convert_call_func import ( ConversionOptions, CONVERSION_OPTIONS, ) -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( +from .dy2static.logging_utils import ( set_code_level, set_verbosity, ) -from paddle.jit.dy2static.program_translator import ( +from .dy2static.program_translator import ( ProgramTranslator, StaticFunction, unwrap_decorators, diff --git a/python/paddle/jit/dy2static/__init__.py b/python/paddle/jit/dy2static/__init__.py index 605c3ea93b..c42116c210 100644 --- a/python/paddle/jit/dy2static/__init__.py +++ b/python/paddle/jit/dy2static/__init__.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .base import saw -from .base import UndefinedVar +from .utils import ( + saw, + UndefinedVar, +) from .convert_operators import convert_logical_and as And # noqa: F401 from .convert_operators import convert_var_dtype as AsDtype # noqa: F401 from .convert_operators import convert_assert as Assert # noqa: F401 @@ -35,5 +37,6 @@ from .convert_operators import convert_shape_compare # noqa: F401 from .assert_transformer import AssertTransformer from .ast_transformer import DygraphToStaticAst from .program_translator import convert_to_static +from .static_analysis import * # noqa: F403 __all__ = [] diff --git a/python/paddle/jit/dy2static/assert_transformer.py b/python/paddle/jit/dy2static/assert_transformer.py index 3a8a8b01ae..81bad1111b 100644 --- a/python/paddle/jit/dy2static/assert_transformer.py +++ b/python/paddle/jit/dy2static/assert_transformer.py @@ -14,10 +14,10 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import ast_to_source_code from .base_transformer import ( BaseTransformer, ) diff --git a/python/paddle/jit/dy2static/ast_transformer.py b/python/paddle/jit/dy2static/ast_transformer.py index b23a6dc368..2e244d6f34 100644 --- a/python/paddle/jit/dy2static/ast_transformer.py +++ b/python/paddle/jit/dy2static/ast_transformer.py @@ -61,7 +61,7 @@ from .return_transformer import ( from .create_variable_transformer import ( CreateVariableTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from .static_analysis import ( StaticAnalysisVisitor, ) from .tensor_shape_transformer import ( @@ -71,8 +71,8 @@ from .decorator_transformer import ( DecoratorTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from . import logging_utils +from .utils import ast_to_source_code __all__ = ['DygraphToStaticAst'] diff --git a/python/paddle/jit/dy2static/base.py b/python/paddle/jit/dy2static/base.py deleted file mode 100644 index 5d1bb01cf5..0000000000 --- a/python/paddle/jit/dy2static/base.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ...fluid.dygraph.dygraph_to_static.utils import saw # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.utils import UndefinedVar # noqa: F401 - -__all__ = [] diff --git a/python/paddle/jit/dy2static/base_transformer.py b/python/paddle/jit/dy2static/base_transformer.py index 518805250d..166753d05f 100644 --- a/python/paddle/jit/dy2static/base_transformer.py +++ b/python/paddle/jit/dy2static/base_transformer.py @@ -14,7 +14,7 @@ from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( ORIGI_INFO, FOR_ITER_INDEX_PREFIX, FOR_ITER_VAR_LEN_PREFIX, diff --git a/python/paddle/jit/dy2static/basic_api_transformer.py b/python/paddle/jit/dy2static/basic_api_transformer.py index 8a80fc4e6e..89fa0738b9 100644 --- a/python/paddle/jit/dy2static/basic_api_transformer.py +++ b/python/paddle/jit/dy2static/basic_api_transformer.py @@ -15,14 +15,13 @@ import astor from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from .static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static import utils +from . import utils from .base_transformer import ( BaseTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code class BasicApiTransformer(BaseTransformer): @@ -166,7 +165,7 @@ class AttributeJstTransformer(BaseTransformer): node = ( gast.parse( "_jst.Attr({}, \"{}\")".format( - ast_to_source_code(value).strip(), attr + utils.ast_to_source_code(value).strip(), attr ) ) .body[0] diff --git a/python/paddle/jit/dy2static/break_continue_transformer.py b/python/paddle/jit/dy2static/break_continue_transformer.py index 23576fdf98..670cc842f3 100644 --- a/python/paddle/jit/dy2static/break_continue_transformer.py +++ b/python/paddle/jit/dy2static/break_continue_transformer.py @@ -15,9 +15,9 @@ from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list -from paddle.fluid.dygraph.dygraph_to_static.utils import BaseNodeVisitor -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( +from paddle.jit.dy2static.utils import index_in_list +from paddle.jit.dy2static.utils import BaseNodeVisitor +from paddle.jit.dy2static.variable_trans_func import ( create_bool_node, ) from .base_transformer import ( diff --git a/python/paddle/jit/dy2static/call_transformer.py b/python/paddle/jit/dy2static/call_transformer.py index 7380934d47..11f0f6624e 100644 --- a/python/paddle/jit/dy2static/call_transformer.py +++ b/python/paddle/jit/dy2static/call_transformer.py @@ -14,11 +14,11 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api +from paddle.jit.dy2static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import is_paddle_api from .base_transformer import ( BaseTransformer, ) diff --git a/python/paddle/jit/dy2static/cast_transformer.py b/python/paddle/jit/dy2static/cast_transformer.py index ca1bf11c43..96a504d70c 100644 --- a/python/paddle/jit/dy2static/cast_transformer.py +++ b/python/paddle/jit/dy2static/cast_transformer.py @@ -14,10 +14,10 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import ast_to_source_code from .base_transformer import ( BaseTransformer, ) diff --git a/python/paddle/jit/dy2static/convert_call_func.py b/python/paddle/jit/dy2static/convert_call_func.py index 325ffb2060..e0f393028c 100644 --- a/python/paddle/jit/dy2static/convert_call_func.py +++ b/python/paddle/jit/dy2static/convert_call_func.py @@ -32,11 +32,11 @@ from .convert_operators import ( convert_enumerate, ) -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( +from paddle.jit.dy2static.logging_utils import ( TranslatorLogger, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_func, unwrap +from paddle.jit.dy2static.utils import is_paddle_func, unwrap from paddle.fluid.dygraph.layers import Layer __all__ = ["convert_call"] diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py index e5b35d0b4c..3643da7591 100644 --- a/python/paddle/jit/dy2static/convert_operators.py +++ b/python/paddle/jit/dy2static/convert_operators.py @@ -15,7 +15,7 @@ import re import paddle from paddle.fluid.data_feeder import convert_dtype -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( +from paddle.jit.dy2static.variable_trans_func import ( to_static_variable, ) from paddle.fluid.framework import core, Variable @@ -46,11 +46,11 @@ from paddle.fluid.layers.control_flow import ( from .return_transformer import ( RETURN_NO_VALUE_VAR_NAME, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( UndefinedVar, Dygraph2StaticException, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper +from paddle.jit.dy2static.utils import GetterSetterHelper from paddle.fluid.layers.utils import copy_mutable_vars diff --git a/python/paddle/jit/dy2static/create_variable_transformer.py b/python/paddle/jit/dy2static/create_variable_transformer.py index feccbfe594..808a047c3a 100644 --- a/python/paddle/jit/dy2static/create_variable_transformer.py +++ b/python/paddle/jit/dy2static/create_variable_transformer.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( FunctionNameLivenessAnalysis, ) -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( +from paddle.jit.dy2static.variable_trans_func import ( create_undefined_var, ) from .base_transformer import ( diff --git a/python/paddle/jit/dy2static/decorator_transformer.py b/python/paddle/jit/dy2static/decorator_transformer.py index f802db72de..cd33f8625a 100644 --- a/python/paddle/jit/dy2static/decorator_transformer.py +++ b/python/paddle/jit/dy2static/decorator_transformer.py @@ -14,13 +14,13 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) from .base_transformer import ( BaseTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( RE_PYNAME, RE_PYMODULE, ast_to_source_code, diff --git a/python/paddle/jit/dy2static/early_return_transformer.py b/python/paddle/jit/dy2static/early_return_transformer.py index 53bb2394e8..72076fb3cd 100644 --- a/python/paddle/jit/dy2static/early_return_transformer.py +++ b/python/paddle/jit/dy2static/early_return_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) from .base_transformer import ( diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/error.py b/python/paddle/jit/dy2static/error.py similarity index 98% rename from python/paddle/fluid/dygraph/dygraph_to_static/error.py rename to python/paddle/jit/dy2static/error.py index f4a6610080..40f5b16def 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/error.py +++ b/python/paddle/jit/dy2static/error.py @@ -17,17 +17,18 @@ import sys import traceback import linecache import re -import numpy as np +import numpy as np # noqa: F401 -from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( +from .origin_info import ( Location, OriginInfo, global_origin_info_map, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( - _is_api_in_module_helper, - RE_PYMODULE, -) +from .utils import _is_api_in_module_helper # noqa: F401 +from .utils import RE_PYMODULE + + +__all__ = [] ERROR_DATA = "Error data about original source code information and traceback." diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py b/python/paddle/jit/dy2static/function_spec.py similarity index 97% rename from python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py rename to python/paddle/jit/dy2static/function_spec.py index cc77e05dad..370fb36bcf 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py +++ b/python/paddle/jit/dy2static/function_spec.py @@ -22,13 +22,16 @@ from paddle.fluid.dygraph import layers from paddle.fluid.layers.utils import flatten from paddle.fluid.layers.utils import pack_sequence_as from paddle.fluid.dygraph.base import switch_to_static_graph -from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs -from paddle.fluid.dygraph.dygraph_to_static.utils import parse_varargs_name -from paddle.fluid.dygraph.dygraph_to_static.utils import type_name -from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code from paddle.fluid.dygraph.io import TranslatedLayer +from . import logging_utils +from .utils import ( + parse_arg_and_kwargs, + parse_varargs_name, + type_name, + func_to_source_code, +) + class FunctionSpec: """ diff --git a/python/paddle/jit/dy2static/ifelse_transformer.py b/python/paddle/jit/dy2static/ifelse_transformer.py index 8bae82c11a..8759e742ce 100644 --- a/python/paddle/jit/dy2static/ifelse_transformer.py +++ b/python/paddle/jit/dy2static/ifelse_transformer.py @@ -22,27 +22,27 @@ from collections import defaultdict from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( create_funcDef_node, ast_to_source_code, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( FunctionNameLivenessAnalysis, ) -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( create_nonlocal_stmt_nodes, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( create_get_args_node, create_set_args_node, ) from .base_transformer import ( BaseTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( FOR_ITER_INDEX_PREFIX, FOR_ITER_TUPLE_PREFIX, FOR_ITER_TUPLE_INDEX_PREFIX, @@ -52,7 +52,7 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import ( FOR_ITER_TARGET_PREFIX, FOR_ITER_ITERATOR_PREFIX, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( GetterSetterHelper, create_name_str, ) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py b/python/paddle/jit/dy2static/logging_utils.py similarity index 99% rename from python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py rename to python/paddle/jit/dy2static/logging_utils.py index 7b004964a4..ca36cb0b35 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py +++ b/python/paddle/jit/dy2static/logging_utils.py @@ -16,7 +16,7 @@ import os import threading from paddle.fluid import log_helper -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from .utils import ast_to_source_code __all__ = ["TranslatorLogger", "set_verbosity", "set_code_level"] diff --git a/python/paddle/jit/dy2static/logical_transformer.py b/python/paddle/jit/dy2static/logical_transformer.py index 1ab4491d8d..cd4d27ac73 100644 --- a/python/paddle/jit/dy2static/logical_transformer.py +++ b/python/paddle/jit/dy2static/logical_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import ast_to_source_code from .base_transformer import ( BaseTransformer, ) diff --git a/python/paddle/jit/dy2static/loop_transformer.py b/python/paddle/jit/dy2static/loop_transformer.py index 7d42638b9e..4bb99e830b 100644 --- a/python/paddle/jit/dy2static/loop_transformer.py +++ b/python/paddle/jit/dy2static/loop_transformer.py @@ -17,21 +17,21 @@ from paddle.utils import gast from collections import defaultdict from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import NodeVarType +from paddle.jit.dy2static.static_analysis import ( StaticAnalysisVisitor, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import get_attribute_full_name +from paddle.jit.dy2static.utils import ( create_nonlocal_stmt_nodes, create_get_args_node, create_set_args_node, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( FunctionNameLivenessAnalysis, ) from .ifelse_transformer import ARGS_NAME @@ -41,7 +41,7 @@ from .base_transformer import ( ForNodeVisitor, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from paddle.jit.dy2static.utils import ( GetterSetterHelper, create_name_str, ) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/jit/dy2static/origin_info.py similarity index 98% rename from python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py rename to python/paddle/jit/dy2static/origin_info.py index 7eb9da1206..7f3c7f719a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/jit/dy2static/origin_info.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import inspect from paddle.utils import gast from paddle.fluid import core -from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap -from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO +from .utils import ( + unwrap, + ORIGI_INFO, +) from paddle.fluid.framework import Program from collections.abc import Sequence diff --git a/python/paddle/jit/dy2static/partial_program.py b/python/paddle/jit/dy2static/partial_program.py index ad5afaff7c..d1ebdbe5cc 100644 --- a/python/paddle/jit/dy2static/partial_program.py +++ b/python/paddle/jit/dy2static/partial_program.py @@ -22,7 +22,7 @@ from paddle.fluid.executor import ( ) from paddle.fluid.dygraph import layers from paddle.fluid.dygraph.base import switch_to_static_graph -from paddle.fluid.dygraph.dygraph_to_static import logging_utils +from . import logging_utils from .return_transformer import ( RETURN_NO_VALUE_MAGIC_NUM, ) diff --git a/python/paddle/jit/dy2static/print_transformer.py b/python/paddle/jit/dy2static/print_transformer.py index cd4f13f019..aa4bc2c219 100644 --- a/python/paddle/jit/dy2static/print_transformer.py +++ b/python/paddle/jit/dy2static/print_transformer.py @@ -14,7 +14,7 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, StaticAnalysisVisitor, ) diff --git a/python/paddle/jit/dy2static/program_translator.py b/python/paddle/jit/dy2static/program_translator.py index 0e912954ae..37a85fc078 100644 --- a/python/paddle/jit/dy2static/program_translator.py +++ b/python/paddle/jit/dy2static/program_translator.py @@ -26,39 +26,35 @@ from paddle.fluid.data_feeder import check_type from paddle.fluid.layers.utils import flatten from paddle.fluid.dygraph.base import param_guard from paddle.fluid.dygraph.base import switch_to_static_graph -from paddle.fluid.dygraph.dygraph_to_static import error -from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( +from . import error +from . import logging_utils +from .origin_info import ( attach_origin_info, -) -from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( create_and_update_origin_info_map, -) -from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( update_op_callstack_with_origin_info, ) + from .partial_program import ( partial_program_from, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import input_specs_compatible -from paddle.fluid.dygraph.dygraph_to_static.utils import type_name -from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap -from paddle.fluid.dygraph.dygraph_to_static.utils import ( +from .utils import ( + ast_to_func, + ast_to_source_code, + func_to_source_code, + input_specs_compatible, + type_name, + unwrap, make_hashable, ALREADY_D2S, ) -from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( +from .function_spec import ( FunctionSpec, _hash_spec_names, -) -from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( get_buffers, get_parameters, ) + from .ast_transformer import DygraphToStaticAst __all__ = ['ProgramTranslator', 'convert_to_static'] diff --git a/python/paddle/jit/dy2static/return_transformer.py b/python/paddle/jit/dy2static/return_transformer.py index c8114e078d..8aa96b6157 100644 --- a/python/paddle/jit/dy2static/return_transformer.py +++ b/python/paddle/jit/dy2static/return_transformer.py @@ -15,16 +15,16 @@ from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list +from paddle.jit.dy2static.utils import index_in_list from .break_continue_transformer import ( ForToWhileTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code +from paddle.jit.dy2static.utils import ast_to_source_code from .base_transformer import ( BaseTransformer, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException -from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO +from paddle.jit.dy2static.utils import Dygraph2StaticException +from paddle.jit.dy2static.utils import ORIGI_INFO __all__ = [ 'RETURN_NO_VALUE_MAGIC_NUM', diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/jit/dy2static/static_analysis.py similarity index 100% rename from python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py rename to python/paddle/jit/dy2static/static_analysis.py diff --git a/python/paddle/jit/dy2static/tensor_shape_transformer.py b/python/paddle/jit/dy2static/tensor_shape_transformer.py index 9dae08b123..ffdba7e790 100644 --- a/python/paddle/jit/dy2static/tensor_shape_transformer.py +++ b/python/paddle/jit/dy2static/tensor_shape_transformer.py @@ -14,8 +14,8 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.utils import ast_to_source_code +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) from .base_transformer import ( diff --git a/python/paddle/jit/dy2static/typehint_transformer.py b/python/paddle/jit/dy2static/typehint_transformer.py index dd272f7647..d5c23d1d7d 100644 --- a/python/paddle/jit/dy2static/typehint_transformer.py +++ b/python/paddle/jit/dy2static/typehint_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( +from paddle.jit.dy2static.static_analysis import ( AstNodeWrapper, ) from .base_transformer import ( diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/jit/dy2static/utils.py similarity index 98% rename from python/paddle/fluid/dygraph/dygraph_to_static/utils.py rename to python/paddle/jit/dy2static/utils.py index 23579720f3..a57134411f 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/jit/dy2static/utils.py @@ -16,7 +16,6 @@ import ast import astor import atexit import copy -import collections from paddle.utils import gast import inspect import os @@ -32,15 +31,17 @@ from paddle.fluid.data_feeder import convert_dtype from paddle.fluid import core from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.layers import assign -import collections from functools import reduce import warnings + +__all__ = [] + # Note(Aurelius): Do not forget the dot `.` to distinguish other # module such as paddlenlp. PADDLE_MODULE_PREFIX = 'paddle.' DYGRAPH_MODULE_PREFIX = 'paddle.fluid.dygraph' -DYGRAPH_TO_STATIC_MODULE_PREFIX = 'paddle.fluid.dygraph.dygraph_to_static' +DYGRAPH_TO_STATIC_MODULE_PREFIX = 'paddle.jit.dy2static' GET_ARGS_FUNC_PREFIX = 'get_args' SET_ARGS_FUNC_PREFIX = 'set_args' ALREADY_D2S = '__already_d2s' @@ -258,19 +259,13 @@ def is_api_in_module(node, module_prefix): func_str = astor.to_source(gast.gast_to_ast(func_node)).strip() try: - # TODO(liym27): - # Consider a better to import modules like: - # source_file = inspect.getfile(dyfunc) - # import_statements = ImportVisitor(source_file).transform() - # import_str = "".join(import_statements) - import paddle - import paddle.fluid as fluid - import paddle.fluid.dygraph as dygraph - import paddle.fluid.layers as layers - import paddle.jit.dy2static as _jst - - from paddle.fluid.dygraph import to_variable - from paddle import to_tensor + import paddle # noqa: F401 + import paddle.fluid as fluid # noqa: F401 + import paddle.fluid.dygraph as dygraph # noqa: F401 + import paddle.fluid.layers as layers # noqa: F401 + import paddle.jit.dy2static as _jst # noqa: F401 + from paddle.fluid.dygraph import to_variable # noqa: F401 + from paddle import to_tensor # noqa: F401 return eval( "_is_api_in_module_helper({}, '{}')".format(func_str, module_prefix) @@ -304,7 +299,7 @@ def is_numpy_api(node): assert isinstance(node, gast.Call), "Input non-Call node for is_numpy_api" func_str = astor.to_source(gast.gast_to_ast(node.func)) try: - import numpy as np + import numpy as np # noqa: F401 module_result = eval( "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") @@ -321,7 +316,7 @@ def is_numpy_api(node): def _delete_keywords_from(node): assert isinstance(node, gast.Call) func_src = astor.to_source(gast.gast_to_ast(node.func)) - import paddle.fluid as fluid + import paddle.fluid as fluid # noqa: F401 full_args = eval(f"inspect.getfullargspec({func_src})") full_args_name = full_args[0] @@ -402,7 +397,7 @@ def update_args_of_func(node, dygraph_node, method_name): ) class_src = astor.to_source(gast.gast_to_ast(dygraph_node.func)) - import paddle.fluid as fluid + import paddle.fluid as fluid # noqa: F401 if method_name == "__init__" or eval( "issubclass({}, fluid.dygraph.Layer)".format(class_src) @@ -894,7 +889,7 @@ class IsControlFlowVisitor(gast.NodeVisitor): return node def _is_node_with_tensor(self, node, name_id): - from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + from paddle.jit.dy2static.static_analysis import ( NodeVarType, ) @@ -1213,7 +1208,6 @@ class FunctionNameLivenessAnalysis(gast.NodeVisitor): because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. """ from paddle.jit.dy2static.loop_transformer import ( - WHILE_CONDITION_PREFIX, WHILE_BODY_PREFIX, FOR_CONDITION_PREFIX, FOR_BODY_PREFIX, diff --git a/python/paddle/jit/dy2static/variable_trans_func.py b/python/paddle/jit/dy2static/variable_trans_func.py index 88f8cd3c2c..c98823242b 100644 --- a/python/paddle/jit/dy2static/variable_trans_func.py +++ b/python/paddle/jit/dy2static/variable_trans_func.py @@ -12,9 +12,82 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import ( # noqa: F401 - create_bool_as_type, - to_static_variable, +import paddle +from paddle.utils import gast +from paddle.fluid.framework import Variable +from paddle.jit.dy2static.utils import ( + UndefinedVar, + create_undefined_variable, ) +from paddle.fluid.layers.utils import map_structure, is_sequence -__all__ = [] +__all__ = [ + 'create_bool_as_type', + 'create_fill_constant_node', + 'to_static_variable', + 'create_undefined_var', +] + + +def create_undefined_var(name): + func_code = "{} = _jst.UndefinedVar('{}')".format(name, name) + return gast.parse(func_code).body[0] + + +def create_fill_constant_node(name, value=0): + func_code = "{} = paddle.full(shape=[1], ".format(name) + if isinstance(value, bool): + func_code += "dtype='bool', fill_value={}, name='{}')".format( + value, name + ) + return gast.parse(func_code).body[0] + if isinstance(value, float): + func_code += "dtype='float64', fill_value={}, name='{}')".format( + value, name + ) + return gast.parse(func_code).body[0] + + if isinstance(value, int): + func_code += "dtype='int64', fill_value={}, name='{}')".format( + value, name + ) + return gast.parse(func_code).body[0] + + +def to_static_variable(x): + ''' + Translate a Python Tensor to PaddlePaddle static graph Tensor + ''' + if isinstance(x, bool): + return paddle.full(shape=[1], dtype='bool', fill_value=x) + if isinstance(x, float): + return paddle.full(shape=[1], dtype='float64', fill_value=x) + if isinstance(x, int): + return paddle.full(shape=[1], dtype='int64', fill_value=x) + if isinstance(x, UndefinedVar) or x is None: + """ + for early return case, we need a variable to represent None, current we use data_layer_not_check. + """ + return create_undefined_variable() + if is_sequence(x): + return map_structure(to_static_variable, x) + return x + + +def create_bool_as_type(x, value=True): + ''' + Create a bool variable, which type is the same as x. + ''' + if isinstance(x, Variable): + return paddle.full(shape=[1], fill_value=value, dtype="bool") + else: + return value + + +def create_bool_node(name, value): + ''' + Create a assign stmt for name = value . + ''' + assert isinstance(value, bool) + node = "{} = {}".format(name, value) + return gast.parse(node).body[0] diff --git a/python/setup.py.in b/python/setup.py.in index cab42d8f36..faa994d744 100755 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -331,7 +331,6 @@ packages=['paddle', 'paddle.inference.contrib.utils', 'paddle.fluid', 'paddle.fluid.dygraph', - 'paddle.fluid.dygraph.dygraph_to_static', 'paddle.fluid.dygraph.amp', 'paddle.fluid.proto', 'paddle.fluid.proto.profiler', -- GitLab