From a9ee3833e548b3950a07aa0c725d240e9f371aee Mon Sep 17 00:00:00 2001 From: 0x45f <23097963+0x45f@users.noreply.github.com> Date: Thu, 5 Aug 2021 17:36:48 +0800 Subject: [PATCH] [Dy2st]Integrated gast library to fix compatibility problem permanently (#34556) * integrated gast library * integrated gast library * fix unittest and remove ast2.py * remove 'gast' from __all__ in __init__.py * add copyright in other files * fix copyright --- .../dygraph_to_static/assert_transformer.py | 2 +- .../dygraph_to_static/ast_transformer.py | 2 +- .../basic_api_transformer.py | 2 +- .../break_continue_transformer.py | 2 +- .../dygraph_to_static/call_transformer.py | 2 +- .../dygraph_to_static/cast_transformer.py | 2 +- .../dygraph_to_static/grad_transformer.py | 2 +- .../dygraph_to_static/ifelse_transformer.py | 2 +- .../dygraph_to_static/list_transformer.py | 2 +- .../dygraph_to_static/logical_transformer.py | 2 +- .../dygraph_to_static/loop_transformer.py | 2 +- .../dygraph/dygraph_to_static/origin_info.py | 2 +- .../dygraph_to_static/print_transformer.py | 2 +- .../dygraph_to_static/program_translator.py | 2 +- .../dygraph_to_static/return_transformer.py | 2 +- .../dygraph_to_static/static_analysis.py | 2 +- .../tensor_shape_transformer.py | 2 +- .../fluid/dygraph/dygraph_to_static/utils.py | 2 +- .../dygraph_to_static/variable_trans_func.py | 2 +- .../dygraph_to_static/test_ast_util.py | 2 +- .../test_basic_api_transformation.py | 2 +- .../dygraph_to_static/test_ifelse_basic.py | 2 +- .../dygraph_to_static/test_logging_utils.py | 2 +- .../dygraph_to_static/test_logical.py | 2 +- .../unittests/dygraph_to_static/test_loop.py | 2 +- .../test_program_translator.py | 2 +- .../dygraph_to_static/test_static_analysis.py | 2 +- .../test_variable_trans_func.py | 2 +- .../unittests/test_gast_with_compatibility.py | 2 +- python/paddle/utils/__init__.py | 1 + python/paddle/utils/gast/__init__.py | 33 + python/paddle/utils/gast/ast3.py | 449 +++++++++++++ python/paddle/utils/gast/astn.py | 64 ++ python/paddle/utils/gast/gast.py | 609 ++++++++++++++++++ python/requirements.txt | 2 - python/setup.py.in | 1 + 36 files changed, 1186 insertions(+), 31 deletions(-) create mode 100644 python/paddle/utils/gast/__init__.py create mode 100644 python/paddle/utils/gast/ast3.py create mode 100644 python/paddle/utils/gast/astn.py create mode 100644 python/paddle/utils/gast/gast.py diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py index fe70fd1094f..e2fcf4f2c27 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py index 29eee429ef6..74f946acedb 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py @@ -18,7 +18,7 @@ from __future__ import print_function # It provides a compatibility layer between the AST of various Python versions, # as produced by ast.parse from the standard ast module. # See details in https://github.com/serge-sans-paille/gast/ -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import AssertTransformer from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py index 5ea1fdfac09..acf2c3ec09b 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. import astor -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static import utils diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py index cb0383b9f73..401ad1c8e84 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py index c2481d16825..3e606139245 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py index 1171b5dbdfa..ef2d062d2d0 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py index 272d480c5b7..98045b3aae4 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import warnings from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py index 5bc1c3d96d9..8fc5a691d21 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py @@ -22,7 +22,7 @@ from collections import defaultdict # It provides a compatibility layer between the AST of various Python versions, # as produced by ast.parse from the standard ast module. # See details in https://github.com/serge-sans-paille/gast/ -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py index a3311765a99..e041fe7c9ac 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import astor -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py index 8470e895dd3..e5c093f9a92 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code cmpop_type_to_str = { diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py index 14bb54983b5..9859feb9d90 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import copy -import gast +from paddle.utils import gast from collections import defaultdict from paddle.fluid import unique_name diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index b2f4060b106..0670c048c5e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -17,7 +17,7 @@ from __future__ import print_function import collections import inspect -import gast +from paddle.utils import gast from paddle.fluid import core from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py index 9d1ec35764b..7960617369e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index 3664c4b0016..58aac8e266f 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -15,7 +15,7 @@ from __future__ import print_function import collections -import gast +from paddle.utils import gast import inspect import six import textwrap diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py index 4bcd49dc8e1..0c7a8bf421a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index cbe6b8a0ff9..ce5f50137b7 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from .utils import is_paddle_api, is_dygraph_api, is_numpy_api, index_in_list __all__ = ['AstNodeWrapper', 'NodeVarType', 'StaticAnalysisVisitor'] diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py index eb53d7ec9be..0bc167132e3 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import copy -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 351a9dcfa3a..650857eefb3 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -19,7 +19,7 @@ import astor import atexit import copy import collections -import gast +from paddle.utils import gast import inspect import os import six diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py index c7844f160ce..b118eeadf7e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py @@ -15,7 +15,7 @@ from __future__ import print_function import six -import gast +from paddle.utils import gast from paddle.fluid import core from paddle.fluid import unique_name diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py index 62b6ac171a4..31a50226f0b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py @@ -16,7 +16,7 @@ from __future__ import print_function import unittest import textwrap -import gast +from paddle.utils import gast import inspect import numpy as np import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index ea745ad6614..b86b85bb90f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -17,7 +17,7 @@ from __future__ import print_function import numpy as np import unittest import inspect -import gast +from paddle.utils import gast import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py index 7ea6aa8907c..975797a487b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py @@ -16,7 +16,7 @@ from __future__ import print_function import unittest import textwrap -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import get_name_ids from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py index 2ed2a273341..385b7ce204a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py @@ -20,7 +20,7 @@ import os import sys import unittest -import gast +from paddle.utils import gast import paddle from paddle.fluid.dygraph.dygraph_to_static import logging_utils diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py index c7193eb2a77..b11e9441c8c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py @@ -18,7 +18,7 @@ from __future__ import print_function import unittest -import gast +from paddle.utils import gast import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py index fe86d5d6368..8116c04f203 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import inspect import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index 2ea3e369099..9e12b6fa208 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -15,7 +15,7 @@ from __future__ import print_function import astor -import gast +from paddle.utils import gast import inspect import numpy as np import textwrap diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py index 0fffb0c9853..7f6d6cf1f3b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import inspect import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py index 9f677d765f9..3431c6aac4c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py index 17ba6869534..8404c563274 100644 --- a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py +++ b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py @@ -15,7 +15,7 @@ from __future__ import print_function import ast -import gast +from paddle.utils import gast import sys import textwrap import unittest diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index c23841ea8b8..2c7bca71698 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from . import gast from .profiler import ProfilerOptions # noqa: F401 from .profiler import Profiler # noqa: F401 from .profiler import get_profiler # noqa: F401 diff --git a/python/paddle/utils/gast/__init__.py b/python/paddle/utils/gast/__init__.py new file mode 100644 index 00000000000..0bcbf5abb81 --- /dev/null +++ b/python/paddle/utils/gast/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +from .gast import * +from ast import NodeVisitor, NodeTransformer, iter_fields, dump diff --git a/python/paddle/utils/gast/ast3.py b/python/paddle/utils/gast/ast3.py new file mode 100644 index 00000000000..58840d5c290 --- /dev/null +++ b/python/paddle/utils/gast/ast3.py @@ -0,0 +1,449 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +from .astn import AstToGAst, GAstToAst +from . import gast +import ast +import sys + + +class Ast3ToGAst(AstToGAst): + if sys.version_info.minor < 9: + + def visit_ExtSlice(self, node): + new_node = gast.Tuple(self._visit(node.dims), gast.Load()) + gast.copy_location(new_node, node) + return new_node + + def visit_Index(self, node): + return self._visit(node.value) + + if sys.version_info.minor < 8: + + def visit_Module(self, node): + new_node = gast.Module( + self._visit(node.body), + [] # type_ignores + ) + return new_node + + def visit_Num(self, node): + new_node = gast.Constant( + node.n, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_Ellipsis(self, node): + new_node = gast.Constant( + Ellipsis, + None, ) + gast.copy_location(new_node, node) + new_node.end_lineno = new_node.end_col_offset = None + return new_node + + def visit_Str(self, node): + new_node = gast.Constant( + node.s, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_Bytes(self, node): + new_node = gast.Constant( + node.s, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_FunctionDef(self, node): + new_node = gast.FunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncFunctionDef(self, node): + new_node = gast.AsyncFunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_For(self, node): + new_node = gast.For( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncFor(self, node): + new_node = gast.AsyncFor( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_With(self, node): + new_node = gast.With( + self._visit(node.items), + self._visit(node.body), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncWith(self, node): + new_node = gast.AsyncWith( + self._visit(node.items), + self._visit(node.body), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_Call(self, node): + if sys.version_info.minor < 5: + if node.starargs: + star = gast.Starred(self._visit(node.starargs), gast.Load()) + gast.copy_location(star, node) + starred = [star] + else: + starred = [] + + if node.kwargs: + kw = gast.keyword(None, self._visit(node.kwargs)) + gast.copy_location(kw, node.kwargs) + kwargs = [kw] + else: + kwargs = [] + else: + starred = kwargs = [] + + new_node = gast.Call( + self._visit(node.func), + self._visit(node.args) + starred, + self._visit(node.keywords) + kwargs, ) + gast.copy_location(new_node, node) + return new_node + + def visit_NameConstant(self, node): + if node.value is None: + new_node = gast.Constant(None, None) + elif node.value is True: + new_node = gast.Constant(True, None) + elif node.value is False: + new_node = gast.Constant(False, None) + gast.copy_location(new_node, node) + return new_node + + def visit_arguments(self, node): + new_node = gast.arguments( + self._visit(node.args), + [], # posonlyargs + self._visit(node.vararg), + self._visit(node.kwonlyargs), + self._visit(node.kw_defaults), + self._visit(node.kwarg), + self._visit(node.defaults), ) + gast.copy_location(new_node, node) + return new_node + + def visit_Name(self, node): + new_node = gast.Name( + self._visit(node.id), + self._visit(node.ctx), + None, + None, ) + ast.copy_location(new_node, node) + return new_node + + def visit_arg(self, node): + if sys.version_info.minor < 8: + extra_args = [None] + else: + extra_args = [self._visit(node.type_comment)] + + new_node = gast.Name( + self._visit(node.arg), + gast.Param(), + self._visit(node.annotation), + *extra_args # type_comment + ) + ast.copy_location(new_node, node) + return new_node + + def visit_ExceptHandler(self, node): + if node.name: + new_node = gast.ExceptHandler( + self._visit(node.type), + gast.Name(node.name, gast.Store(), None, None), + self._visit(node.body)) + ast.copy_location(new_node, node) + return new_node + else: + return self.generic_visit(node) + + if sys.version_info.minor < 6: + + def visit_comprehension(self, node): + new_node = gast.comprehension( + target=self._visit(node.target), + iter=self._visit(node.iter), + ifs=self._visit(node.ifs), + is_async=0, ) + return ast.copy_location(new_node, node) + + +class GAstToAst3(GAstToAst): + if sys.version_info.minor < 9: + + def visit_Subscript(self, node): + def adjust_slice(s): + if isinstance(s, ast.Slice): + return s + else: + return ast.Index(s) + + if isinstance(node.slice, gast.Tuple): + if any(isinstance(elt, gast.slice) for elt in node.slice.elts): + new_slice = ast.ExtSlice([ + adjust_slice(x) for x in self._visit(node.slice.elts) + ]) + else: + value = ast.Tuple(self._visit(node.slice.elts), ast.Load()) + ast.copy_location(value, node.slice) + new_slice = ast.Index(value) + else: + new_slice = adjust_slice(self._visit(node.slice)) + ast.copy_location(new_slice, node.slice) + + new_node = ast.Subscript( + self._visit(node.value), + new_slice, + self._visit(node.ctx), ) + ast.copy_location(new_node, node) + return new_node + + if sys.version_info.minor < 8: + + def visit_Module(self, node): + new_node = ast.Module(self._visit(node.body)) + return new_node + + def visit_Constant(self, node): + if node.value is None: + new_node = ast.NameConstant(node.value) + elif node.value is Ellipsis: + new_node = ast.Ellipsis() + elif isinstance(node.value, bool): + new_node = ast.NameConstant(node.value) + elif isinstance(node.value, (int, float, complex)): + new_node = ast.Num(node.value) + elif isinstance(node.value, str): + new_node = ast.Str(node.value) + else: + new_node = ast.Bytes(node.value) + ast.copy_location(new_node, node) + return new_node + + def _make_arg(self, node): + if node is None: + return None + + if sys.version_info.minor < 8: + extra_args = tuple() + else: + extra_args = self._visit(node.type_comment), + + new_node = ast.arg( + self._visit(node.id), self._visit(node.annotation), *extra_args) + return ast.copy_location(new_node, node) + + def visit_Name(self, node): + new_node = ast.Name( + self._visit(node.id), + self._visit(node.ctx), ) + ast.copy_location(new_node, node) + return new_node + + def visit_ExceptHandler(self, node): + if node.name: + new_node = ast.ExceptHandler( + self._visit(node.type), node.name.id, self._visit(node.body)) + return ast.copy_location(new_node, node) + else: + return self.generic_visit(node) + + if sys.version_info.minor < 5: + + def visit_Call(self, node): + if node.args and isinstance(node.args[-1], gast.Starred): + args = node.args[:-1] + starargs = node.args[-1].value + else: + args = node.args + starargs = None + + if node.keywords and node.keywords[-1].arg is None: + keywords = node.keywords[:-1] + kwargs = node.keywords[-1].value + else: + keywords = node.keywords + kwargs = None + + new_node = ast.Call( + self._visit(node.func), + self._visit(args), + self._visit(keywords), + self._visit(starargs), + self._visit(kwargs), ) + ast.copy_location(new_node, node) + return new_node + + def visit_ClassDef(self, node): + self.generic_visit(node) + new_node = ast.ClassDef( + name=self._visit(node.name), + bases=self._visit(node.bases), + keywords=self._visit(node.keywords), + body=self._visit(node.body), + decorator_list=self._visit(node.decorator_list), + starargs=None, + kwargs=None, ) + return ast.copy_location(new_node, node) + + elif sys.version_info.minor < 8: + + def visit_FunctionDef(self, node): + new_node = ast.FunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncFunctionDef(self, node): + new_node = ast.AsyncFunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), ) + ast.copy_location(new_node, node) + return new_node + + def visit_For(self, node): + new_node = ast.For( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncFor(self, node): + new_node = ast.AsyncFor( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + ast.copy_location(new_node, node) + return new_node + + def visit_With(self, node): + new_node = ast.With( + self._visit(node.items), + self._visit(node.body), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncWith(self, node): + new_node = ast.AsyncWith( + self._visit(node.items), + self._visit(node.body), ) + ast.copy_location(new_node, node) + return new_node + + def visit_Call(self, node): + new_node = ast.Call( + self._visit(node.func), + self._visit(node.args), + self._visit(node.keywords), ) + ast.copy_location(new_node, node) + return new_node + + def visit_arguments(self, node): + extra_args = [ + self._make_arg(node.vararg), + [self._make_arg(n) for n in node.kwonlyargs], + self._visit(node.kw_defaults), + self._make_arg(node.kwarg), + self._visit(node.defaults), + ] + if sys.version_info.minor >= 8: + new_node = ast.arguments( + [self._make_arg(arg) for arg in node.posonlyargs], + [self._make_arg(n) for n in node.args], *extra_args) + else: + new_node = ast.arguments([self._make_arg(n) for n in node.args], + *extra_args) + return new_node + + +def ast_to_gast(node): + return Ast3ToGAst().visit(node) + + +def gast_to_ast(node): + return GAstToAst3().visit(node) diff --git a/python/paddle/utils/gast/astn.py b/python/paddle/utils/gast/astn.py new file mode 100644 index 00000000000..bd88ba5efc5 --- /dev/null +++ b/python/paddle/utils/gast/astn.py @@ -0,0 +1,64 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +import ast +from . import gast + + +def _generate_translators(to): + class Translator(ast.NodeTransformer): + def _visit(self, node): + if isinstance(node, list): + return [self._visit(n) for n in node] + elif isinstance(node, ast.AST): + return self.visit(node) + else: + return node + + def generic_visit(self, node): + cls = type(node).__name__ + # handle nodes that are not part of the AST + if not hasattr(to, cls): + return + new_node = getattr(to, cls)() + for field in node._fields: + setattr(new_node, field, self._visit(getattr(node, field))) + for attr in getattr(node, '_attributes'): + if hasattr(node, attr): + setattr(new_node, attr, getattr(node, attr)) + return new_node + + return Translator + + +AstToGAst = _generate_translators(gast) + +GAstToAst = _generate_translators(ast) diff --git a/python/paddle/utils/gast/gast.py b/python/paddle/utils/gast/gast.py new file mode 100644 index 00000000000..f561c83995a --- /dev/null +++ b/python/paddle/utils/gast/gast.py @@ -0,0 +1,609 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +import sys as _sys +import ast as _ast +from ast import boolop, cmpop, excepthandler, expr, expr_context, operator +from ast import slice, stmt, unaryop, mod, AST +from ast import iter_child_nodes, walk + +try: + from ast import TypeIgnore +except ImportError: + + class TypeIgnore(AST): + pass + + +def _make_node(Name, Fields, Attributes, Bases): + def create_node(self, *args, **kwargs): + nbparam = len(args) + len(kwargs) + assert nbparam in (0, len(Fields)), \ + "Bad argument number for {}: {}, expecting {}".\ + format(Name, nbparam, len(Fields)) + self._fields = Fields + self._attributes = Attributes + for argname, argval in zip(self._fields, args): + setattr(self, argname, argval) + for argname, argval in kwargs.items(): + assert argname in Fields, \ + "Invalid Keyword argument for {}: {}".format(Name, argname) + setattr(self, argname, argval) + + setattr(_sys.modules[__name__], Name, + type(Name, Bases, {'__init__': create_node})) + + +_nodes = ( + # mod + ('Module', (('body', 'type_ignores'), (), (mod, ))), + ('Interactive', (('body', ), (), (mod, ))), + ('Expression', (('body', ), (), (mod, ))), + ('FunctionType', (('argtypes', 'returns'), (), (mod, ))), + ('Suite', (('body', ), (), (mod, ))), + + # stmt + ('FunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', + 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncFunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', + 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('ClassDef', (( + 'name', + 'bases', + 'keywords', + 'body', + 'decorator_list', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Return', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Delete', (('targets', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Assign', (( + 'targets', + 'value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AugAssign', (( + 'target', + 'op', + 'value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AnnAssign', (( + 'target', + 'annotation', + 'value', + 'simple', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Print', (( + 'dest', + 'values', + 'nl', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('For', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncFor', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('While', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('If', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('With', (('items', 'body', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncWith', (('items', 'body', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Raise', (( + 'exc', + 'cause', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Try', (( + 'body', + 'handlers', + 'orelse', + 'finalbody', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Assert', (( + 'test', + 'msg', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Import', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('ImportFrom', (( + 'module', + 'names', + 'level', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Exec', (( + 'body', + 'globals', + 'locals', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Global', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Nonlocal', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Expr', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Pass', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Break', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Continue', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + + # expr + ('BoolOp', (( + 'op', + 'values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('BinOp', (( + 'left', + 'op', + 'right', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('UnaryOp', (( + 'op', + 'operand', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Lambda', (( + 'args', + 'body', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('IfExp', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Dict', (( + 'keys', + 'values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Set', (('elts', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('ListComp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('SetComp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('DictComp', (( + 'key', + 'value', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('GeneratorExp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Await', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Yield', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('YieldFrom', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Compare', (( + 'left', + 'ops', + 'comparators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Call', (( + 'func', + 'args', + 'keywords', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Repr', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('FormattedValue', (( + 'value', + 'conversion', + 'format_spec', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('JoinedStr', (('values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Constant', (('value', 'kind'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Attribute', (( + 'value', + 'attr', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Subscript', (( + 'value', + 'slice', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Starred', (( + 'value', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Name', (('id', 'ctx', 'annotation', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('List', (( + 'elts', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Tuple', (( + 'elts', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + + # expr_context + ('Load', ((), (), (expr_context, ))), + ('Store', ((), (), (expr_context, ))), + ('Del', ((), (), (expr_context, ))), + ('AugLoad', ((), (), (expr_context, ))), + ('AugStore', ((), (), (expr_context, ))), + ('Param', ((), (), (expr_context, ))), + + # slice + ('Slice', (('lower', 'upper', 'step'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (slice, ))), + + # boolop + ('And', ((), (), (boolop, ))), + ('Or', ((), (), (boolop, ))), + + # operator + ('Add', ((), (), (operator, ))), + ('Sub', ((), (), (operator, ))), + ('Mult', ((), (), (operator, ))), + ('MatMult', ((), (), (operator, ))), + ('Div', ((), (), (operator, ))), + ('Mod', ((), (), (operator, ))), + ('Pow', ((), (), (operator, ))), + ('LShift', ((), (), (operator, ))), + ('RShift', ((), (), (operator, ))), + ('BitOr', ((), (), (operator, ))), + ('BitXor', ((), (), (operator, ))), + ('BitAnd', ((), (), (operator, ))), + ('FloorDiv', ((), (), (operator, ))), + + # unaryop + ('Invert', ((), (), ( + unaryop, + AST, ))), + ('Not', ((), (), ( + unaryop, + AST, ))), + ('UAdd', ((), (), ( + unaryop, + AST, ))), + ('USub', ((), (), ( + unaryop, + AST, ))), + + # cmpop + ('Eq', ((), (), (cmpop, ))), + ('NotEq', ((), (), (cmpop, ))), + ('Lt', ((), (), (cmpop, ))), + ('LtE', ((), (), (cmpop, ))), + ('Gt', ((), (), (cmpop, ))), + ('GtE', ((), (), (cmpop, ))), + ('Is', ((), (), (cmpop, ))), + ('IsNot', ((), (), (cmpop, ))), + ('In', ((), (), (cmpop, ))), + ('NotIn', ((), (), (cmpop, ))), + + # comprehension + ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST, ))), + + # excepthandler + ('ExceptHandler', (('type', 'name', 'body'), + ('lineno', 'col_offset', 'end_lineno', + 'end_col_offset'), (excepthandler, ))), + + # arguments + ('arguments', (('args', 'posonlyargs', 'vararg', 'kwonlyargs', + 'kw_defaults', 'kwarg', 'defaults'), (), (AST, ))), + + # keyword + ('keyword', + (('arg', 'value'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), (AST, ))), + + # alias + ('alias', (('name', 'asname'), (), (AST, ))), + + # withitem + ('withitem', (('context_expr', 'optional_vars'), (), (AST, ))), + + # type_ignore + ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore, ))), ) + +for name, descr in _nodes: + _make_node(name, *descr) + +py_version = _sys.version_info.major +if py_version != 3: + raise RuntimeError( + 'Required Python version >= 3, but received Python version == {}'. + format(py_version)) + +from .ast3 import ast_to_gast, gast_to_ast + + +def parse(*args, **kwargs): + return ast_to_gast(_ast.parse(*args, **kwargs)) + + +def literal_eval(node_or_string): + if isinstance(node_or_string, AST): + node_or_string = gast_to_ast(node_or_string) + return _ast.literal_eval(node_or_string) + + +def get_docstring(node, clean=True): + if not isinstance(node, (FunctionDef, ClassDef, Module)): + raise TypeError("%r can't have docstrings" % node.__class__.__name__) + if node.body and isinstance(node.body[0], Expr) and \ + isinstance(node.body[0].value, Constant): + if clean: + import inspect + holder = node.body[0].value + return inspect.cleandoc(getattr(holder, holder._fields[0])) + return node.body[0].value.s + + +# the following are directly imported from python3.8's Lib/ast.py # + + +def copy_location(new_node, old_node): + """ + Copy source location (`lineno`, `col_offset`, `end_lineno`, and + `end_col_offset` attributes) from *old_node* to *new_node* if possible, + and return *new_node*. + """ + for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': + if attr in old_node._attributes and attr in new_node._attributes \ + and hasattr(old_node, attr): + setattr(new_node, attr, getattr(old_node, attr)) + return new_node + + +def fix_missing_locations(node): + """ + When you compile a node tree with compile(), the compiler expects lineno + and col_offset attributes for every node that supports them. This is + rather tedious to fill in for generated nodes, so this helper adds these + attributes recursively where not already set, by setting them to the values + of the parent node. It works recursively starting at *node*. + """ + + def _fix(node, lineno, col_offset, end_lineno, end_col_offset): + if 'lineno' in node._attributes: + if not hasattr(node, 'lineno'): + node.lineno = lineno + else: + lineno = node.lineno + if 'end_lineno' in node._attributes: + if not hasattr(node, 'end_lineno'): + node.end_lineno = end_lineno + else: + end_lineno = node.end_lineno + if 'col_offset' in node._attributes: + if not hasattr(node, 'col_offset'): + node.col_offset = col_offset + else: + col_offset = node.col_offset + if 'end_col_offset' in node._attributes: + if not hasattr(node, 'end_col_offset'): + node.end_col_offset = end_col_offset + else: + end_col_offset = node.end_col_offset + for child in iter_child_nodes(node): + _fix(child, lineno, col_offset, end_lineno, end_col_offset) + + _fix(node, 1, 0, 1, 0) + return node + + +def increment_lineno(node, n=1): + """ + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. + """ + for child in walk(node): + if 'lineno' in child._attributes: + child.lineno = (getattr(child, 'lineno', 0) or 0) + n + if 'end_lineno' in child._attributes: + child.end_lineno = (getattr(child, 'end_lineno', 0) or 0) + n + return node diff --git a/python/requirements.txt b/python/requirements.txt index e9da2aa24d6..42327007615 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -2,8 +2,6 @@ requests>=2.20.0 numpy>=1.13 ; python_version>="3.5" and platform_system != "Windows" numpy>=1.13, <=1.19.3 ; python_version>="3.5" and platform_system == "Windows" protobuf>=3.1.0 -gast>=0.3.3, <=0.4.0 ; platform_system != "Windows" -gast==0.3.3 ; platform_system == "Windows" Pillow six decorator diff --git a/python/setup.py.in b/python/setup.py.in index 64b5dd4cd21..f4535fe157d 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -139,6 +139,7 @@ write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/f packages=['paddle', 'paddle.libs', 'paddle.utils', + 'paddle.utils.gast', 'paddle.utils.cpp_extension', 'paddle.dataset', 'paddle.reader', -- GitLab