diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py index fe70fd1094f581caf0ae028bb63ec1dc155928fc..e2fcf4f2c2712eddc07cf5552738f7ae3aa01e0f 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py index 29eee429ef66ab7e324d234b903d0e80510454b7..74f946acedb27f396296417286bc827d2fb5adea 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py @@ -18,7 +18,7 @@ from __future__ import print_function # It provides a compatibility layer between the AST of various Python versions, # as produced by ast.parse from the standard ast module. # See details in https://github.com/serge-sans-paille/gast/ -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import AssertTransformer from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py index 5ea1fdfac0928ad465fc7e29813fe42182047c6a..acf2c3ec09b5d5cdc1a43e323aa83acda8b1b837 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. import astor -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static import utils diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py index cb0383b9f736235b85735e38635f3db43ab23784..401ad1c8e84e45eced099f7908450837e32c7030 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py index c2481d16825ec8d5ed3903da0414b4b03ae92e03..3e606139245d60425cd488691b7f78b18e9c1ae6 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py index 1171b5dbdfa22a21714de53ace9a8d11283bc013..ef2d062d2d0187de9a030de2a71dff66b7b51aad 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py index 272d480c5b7a208e8e5b19484c48ffd58048c36f..98045b3aae432276a3121634af893040a14a172a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/grad_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import warnings from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py index 5bc1c3d96d9c959fae0c39290bc531921e6022ec..8fc5a691d212c22924574f09b07b2bb448d97541 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py @@ -22,7 +22,7 @@ from collections import defaultdict # It provides a compatibility layer between the AST of various Python versions, # as produced by ast.parse from the standard ast module. # See details in https://github.com/serge-sans-paille/gast/ -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py index a3311765a996f6592a680d2fdb878c13006143c1..e041fe7c9ac37c66b359491379e34d0114413edd 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import astor -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py index 8470e895dd3c89c27ca5a2ed2e95763d37030d1b..e5c093f9a9255c3d20ec2db4bb91571c6cc57d6b 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code cmpop_type_to_str = { diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py index 14bb54983b524ad1c09aa0d66f37b2b2aae6dbe8..9859feb9d9079219f79e4bdbf2bb33e766021fcc 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import copy -import gast +from paddle.utils import gast from collections import defaultdict from paddle.fluid import unique_name diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index b2f4060b106828865d5e2ffc6ce9215cd0c19503..0670c048c5e26b13445d7dd74f2ec21efd5d043e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -17,7 +17,7 @@ from __future__ import print_function import collections import inspect -import gast +from paddle.utils import gast from paddle.fluid import core from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py index 9d1ec35764b090cb786fef261e05eedd7f78247d..7960617369e3f24c7ba134dfb5a2de923afb538c 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index 3664c4b00164499bc8e6f9f4f797124c82d0dea4..58aac8e266fedd0c76a6629d43d8a92f9757b03d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -15,7 +15,7 @@ from __future__ import print_function import collections -import gast +from paddle.utils import gast import inspect import six import textwrap diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py index 4bcd49dc8e157716ac232ba8caf1dfa36d3bf1d1..0c7a8bf421a1282bfa542c8eaf4a93db90a1ad90 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index cbe6b8a0ff942846d9e751a3daa2e02e4eefbb6b..ce5f50137b7aa9797e6ba7ed0845422859f2e68d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast from .utils import is_paddle_api, is_dygraph_api, is_numpy_api, index_in_list __all__ = ['AstNodeWrapper', 'NodeVarType', 'StaticAnalysisVisitor'] diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py index eb53d7ec9bec894771afce2191dfe195fc53580d..0bc167132e3ed76499457703f7263a60393d3af0 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py @@ -15,7 +15,7 @@ from __future__ import print_function import copy -import gast +from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 351a9dcfa3aa2a7db56191f821096c02adaf831d..650857eefb3bb1ba4909e8746d42d380b5170bf9 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -19,7 +19,7 @@ import astor import atexit import copy import collections -import gast +from paddle.utils import gast import inspect import os import six diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py index c7844f160cee5ac59c75071b38696fcc33982946..b118eeadf7e7e5a93b75fa467b46af511156bb23 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py @@ -15,7 +15,7 @@ from __future__ import print_function import six -import gast +from paddle.utils import gast from paddle.fluid import core from paddle.fluid import unique_name diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py index 62b6ac171a4c96089bade7b24ec8f2d02c9d94da..31a50226f0b79e0aed8fe9377b72326dde9d7e61 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py @@ -16,7 +16,7 @@ from __future__ import print_function import unittest import textwrap -import gast +from paddle.utils import gast import inspect import numpy as np import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index ea745ad661425381811b2405362ce254b0403fe1..b86b85bb90ff69082fe45855b47628da2cd5a1dd 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -17,7 +17,7 @@ from __future__ import print_function import numpy as np import unittest import inspect -import gast +from paddle.utils import gast import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py index 7ea6aa8907c282b3b68c88dc3e0b5c1cad344fe1..975797a487be72d0cadfd2bdff1d8e6dac991ccf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse_basic.py @@ -16,7 +16,7 @@ from __future__ import print_function import unittest import textwrap -import gast +from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import get_name_ids from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py index 2ed2a2733418058cb4ac2e64a9adbee282c76e4f..385b7ce204a869190ac9c4e41cc57a9bf66d990f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py @@ -20,7 +20,7 @@ import os import sys import unittest -import gast +from paddle.utils import gast import paddle from paddle.fluid.dygraph.dygraph_to_static import logging_utils diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py index c7193eb2a77bc85429e562000fe4f4101aff1a99..b11e9441c8c0e21ec8babb773fda1f974c1beccd 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py @@ -18,7 +18,7 @@ from __future__ import print_function import unittest -import gast +from paddle.utils import gast import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py index fe86d5d636811e83f7768e5414930e82da46ca1f..8116c04f2034fe3b6d1070f4ef4b067b314be64c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import inspect import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index 2ea3e3690991093c4ee7651d75a7653ace89c107..9e12b6fa208505ef75c80516d1c65d06141a048e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -15,7 +15,7 @@ from __future__ import print_function import astor -import gast +from paddle.utils import gast import inspect import numpy as np import textwrap diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py index 0fffb0c985375be5f126852b8fa890bb0c32ab9d..7f6d6cf1f3b0052d3a6a57ebf719b2db1e0760ee 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import inspect import numpy as np import paddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py index 9f677d765f9ab26ec37b02feebe806799d362576..3431c6aac4cbefd9827181b6a094bfa2655dc627 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py @@ -14,7 +14,7 @@ from __future__ import print_function -import gast +from paddle.utils import gast import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py index 17ba6869534fe72d6d82062d957cf9a546f672aa..8404c563274b1e410e664d354435cc384a038c46 100644 --- a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py +++ b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py @@ -15,7 +15,7 @@ from __future__ import print_function import ast -import gast +from paddle.utils import gast import sys import textwrap import unittest diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index c23841ea8b802bfec28c2b293d6ac0d57f423d5b..2c7bca71698d44034fa2fd06bde0d06cfda40455 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from . import gast from .profiler import ProfilerOptions # noqa: F401 from .profiler import Profiler # noqa: F401 from .profiler import get_profiler # noqa: F401 diff --git a/python/paddle/utils/gast/__init__.py b/python/paddle/utils/gast/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0bcbf5abb81b26c10f619674bd895453ec37bd04 --- /dev/null +++ b/python/paddle/utils/gast/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +from .gast import * +from ast import NodeVisitor, NodeTransformer, iter_fields, dump diff --git a/python/paddle/utils/gast/ast3.py b/python/paddle/utils/gast/ast3.py new file mode 100644 index 0000000000000000000000000000000000000000..58840d5c29074c3c83337defdc8812b6b7d7a4f0 --- /dev/null +++ b/python/paddle/utils/gast/ast3.py @@ -0,0 +1,449 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +from .astn import AstToGAst, GAstToAst +from . import gast +import ast +import sys + + +class Ast3ToGAst(AstToGAst): + if sys.version_info.minor < 9: + + def visit_ExtSlice(self, node): + new_node = gast.Tuple(self._visit(node.dims), gast.Load()) + gast.copy_location(new_node, node) + return new_node + + def visit_Index(self, node): + return self._visit(node.value) + + if sys.version_info.minor < 8: + + def visit_Module(self, node): + new_node = gast.Module( + self._visit(node.body), + [] # type_ignores + ) + return new_node + + def visit_Num(self, node): + new_node = gast.Constant( + node.n, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_Ellipsis(self, node): + new_node = gast.Constant( + Ellipsis, + None, ) + gast.copy_location(new_node, node) + new_node.end_lineno = new_node.end_col_offset = None + return new_node + + def visit_Str(self, node): + new_node = gast.Constant( + node.s, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_Bytes(self, node): + new_node = gast.Constant( + node.s, + None, ) + gast.copy_location(new_node, node) + return new_node + + def visit_FunctionDef(self, node): + new_node = gast.FunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncFunctionDef(self, node): + new_node = gast.AsyncFunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_For(self, node): + new_node = gast.For( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncFor(self, node): + new_node = gast.AsyncFor( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_With(self, node): + new_node = gast.With( + self._visit(node.items), + self._visit(node.body), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_AsyncWith(self, node): + new_node = gast.AsyncWith( + self._visit(node.items), + self._visit(node.body), + None, # type_comment + ) + gast.copy_location(new_node, node) + return new_node + + def visit_Call(self, node): + if sys.version_info.minor < 5: + if node.starargs: + star = gast.Starred(self._visit(node.starargs), gast.Load()) + gast.copy_location(star, node) + starred = [star] + else: + starred = [] + + if node.kwargs: + kw = gast.keyword(None, self._visit(node.kwargs)) + gast.copy_location(kw, node.kwargs) + kwargs = [kw] + else: + kwargs = [] + else: + starred = kwargs = [] + + new_node = gast.Call( + self._visit(node.func), + self._visit(node.args) + starred, + self._visit(node.keywords) + kwargs, ) + gast.copy_location(new_node, node) + return new_node + + def visit_NameConstant(self, node): + if node.value is None: + new_node = gast.Constant(None, None) + elif node.value is True: + new_node = gast.Constant(True, None) + elif node.value is False: + new_node = gast.Constant(False, None) + gast.copy_location(new_node, node) + return new_node + + def visit_arguments(self, node): + new_node = gast.arguments( + self._visit(node.args), + [], # posonlyargs + self._visit(node.vararg), + self._visit(node.kwonlyargs), + self._visit(node.kw_defaults), + self._visit(node.kwarg), + self._visit(node.defaults), ) + gast.copy_location(new_node, node) + return new_node + + def visit_Name(self, node): + new_node = gast.Name( + self._visit(node.id), + self._visit(node.ctx), + None, + None, ) + ast.copy_location(new_node, node) + return new_node + + def visit_arg(self, node): + if sys.version_info.minor < 8: + extra_args = [None] + else: + extra_args = [self._visit(node.type_comment)] + + new_node = gast.Name( + self._visit(node.arg), + gast.Param(), + self._visit(node.annotation), + *extra_args # type_comment + ) + ast.copy_location(new_node, node) + return new_node + + def visit_ExceptHandler(self, node): + if node.name: + new_node = gast.ExceptHandler( + self._visit(node.type), + gast.Name(node.name, gast.Store(), None, None), + self._visit(node.body)) + ast.copy_location(new_node, node) + return new_node + else: + return self.generic_visit(node) + + if sys.version_info.minor < 6: + + def visit_comprehension(self, node): + new_node = gast.comprehension( + target=self._visit(node.target), + iter=self._visit(node.iter), + ifs=self._visit(node.ifs), + is_async=0, ) + return ast.copy_location(new_node, node) + + +class GAstToAst3(GAstToAst): + if sys.version_info.minor < 9: + + def visit_Subscript(self, node): + def adjust_slice(s): + if isinstance(s, ast.Slice): + return s + else: + return ast.Index(s) + + if isinstance(node.slice, gast.Tuple): + if any(isinstance(elt, gast.slice) for elt in node.slice.elts): + new_slice = ast.ExtSlice([ + adjust_slice(x) for x in self._visit(node.slice.elts) + ]) + else: + value = ast.Tuple(self._visit(node.slice.elts), ast.Load()) + ast.copy_location(value, node.slice) + new_slice = ast.Index(value) + else: + new_slice = adjust_slice(self._visit(node.slice)) + ast.copy_location(new_slice, node.slice) + + new_node = ast.Subscript( + self._visit(node.value), + new_slice, + self._visit(node.ctx), ) + ast.copy_location(new_node, node) + return new_node + + if sys.version_info.minor < 8: + + def visit_Module(self, node): + new_node = ast.Module(self._visit(node.body)) + return new_node + + def visit_Constant(self, node): + if node.value is None: + new_node = ast.NameConstant(node.value) + elif node.value is Ellipsis: + new_node = ast.Ellipsis() + elif isinstance(node.value, bool): + new_node = ast.NameConstant(node.value) + elif isinstance(node.value, (int, float, complex)): + new_node = ast.Num(node.value) + elif isinstance(node.value, str): + new_node = ast.Str(node.value) + else: + new_node = ast.Bytes(node.value) + ast.copy_location(new_node, node) + return new_node + + def _make_arg(self, node): + if node is None: + return None + + if sys.version_info.minor < 8: + extra_args = tuple() + else: + extra_args = self._visit(node.type_comment), + + new_node = ast.arg( + self._visit(node.id), self._visit(node.annotation), *extra_args) + return ast.copy_location(new_node, node) + + def visit_Name(self, node): + new_node = ast.Name( + self._visit(node.id), + self._visit(node.ctx), ) + ast.copy_location(new_node, node) + return new_node + + def visit_ExceptHandler(self, node): + if node.name: + new_node = ast.ExceptHandler( + self._visit(node.type), node.name.id, self._visit(node.body)) + return ast.copy_location(new_node, node) + else: + return self.generic_visit(node) + + if sys.version_info.minor < 5: + + def visit_Call(self, node): + if node.args and isinstance(node.args[-1], gast.Starred): + args = node.args[:-1] + starargs = node.args[-1].value + else: + args = node.args + starargs = None + + if node.keywords and node.keywords[-1].arg is None: + keywords = node.keywords[:-1] + kwargs = node.keywords[-1].value + else: + keywords = node.keywords + kwargs = None + + new_node = ast.Call( + self._visit(node.func), + self._visit(args), + self._visit(keywords), + self._visit(starargs), + self._visit(kwargs), ) + ast.copy_location(new_node, node) + return new_node + + def visit_ClassDef(self, node): + self.generic_visit(node) + new_node = ast.ClassDef( + name=self._visit(node.name), + bases=self._visit(node.bases), + keywords=self._visit(node.keywords), + body=self._visit(node.body), + decorator_list=self._visit(node.decorator_list), + starargs=None, + kwargs=None, ) + return ast.copy_location(new_node, node) + + elif sys.version_info.minor < 8: + + def visit_FunctionDef(self, node): + new_node = ast.FunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncFunctionDef(self, node): + new_node = ast.AsyncFunctionDef( + self._visit(node.name), + self._visit(node.args), + self._visit(node.body), + self._visit(node.decorator_list), + self._visit(node.returns), ) + ast.copy_location(new_node, node) + return new_node + + def visit_For(self, node): + new_node = ast.For( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncFor(self, node): + new_node = ast.AsyncFor( + self._visit(node.target), + self._visit(node.iter), + self._visit(node.body), + self._visit(node.orelse), + None, # type_comment + ) + ast.copy_location(new_node, node) + return new_node + + def visit_With(self, node): + new_node = ast.With( + self._visit(node.items), + self._visit(node.body), ) + ast.copy_location(new_node, node) + return new_node + + def visit_AsyncWith(self, node): + new_node = ast.AsyncWith( + self._visit(node.items), + self._visit(node.body), ) + ast.copy_location(new_node, node) + return new_node + + def visit_Call(self, node): + new_node = ast.Call( + self._visit(node.func), + self._visit(node.args), + self._visit(node.keywords), ) + ast.copy_location(new_node, node) + return new_node + + def visit_arguments(self, node): + extra_args = [ + self._make_arg(node.vararg), + [self._make_arg(n) for n in node.kwonlyargs], + self._visit(node.kw_defaults), + self._make_arg(node.kwarg), + self._visit(node.defaults), + ] + if sys.version_info.minor >= 8: + new_node = ast.arguments( + [self._make_arg(arg) for arg in node.posonlyargs], + [self._make_arg(n) for n in node.args], *extra_args) + else: + new_node = ast.arguments([self._make_arg(n) for n in node.args], + *extra_args) + return new_node + + +def ast_to_gast(node): + return Ast3ToGAst().visit(node) + + +def gast_to_ast(node): + return GAstToAst3().visit(node) diff --git a/python/paddle/utils/gast/astn.py b/python/paddle/utils/gast/astn.py new file mode 100644 index 0000000000000000000000000000000000000000..bd88ba5efc512a9b072793a9c595dab066b0296e --- /dev/null +++ b/python/paddle/utils/gast/astn.py @@ -0,0 +1,64 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +import ast +from . import gast + + +def _generate_translators(to): + class Translator(ast.NodeTransformer): + def _visit(self, node): + if isinstance(node, list): + return [self._visit(n) for n in node] + elif isinstance(node, ast.AST): + return self.visit(node) + else: + return node + + def generic_visit(self, node): + cls = type(node).__name__ + # handle nodes that are not part of the AST + if not hasattr(to, cls): + return + new_node = getattr(to, cls)() + for field in node._fields: + setattr(new_node, field, self._visit(getattr(node, field))) + for attr in getattr(node, '_attributes'): + if hasattr(node, attr): + setattr(new_node, attr, getattr(node, attr)) + return new_node + + return Translator + + +AstToGAst = _generate_translators(gast) + +GAstToAst = _generate_translators(ast) diff --git a/python/paddle/utils/gast/gast.py b/python/paddle/utils/gast/gast.py new file mode 100644 index 0000000000000000000000000000000000000000..f561c83995ac1d19d48c5f6851912bafe7d83565 --- /dev/null +++ b/python/paddle/utils/gast/gast.py @@ -0,0 +1,609 @@ +# Copyright (c) 2016, Serge Guelton +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# Neither the name of HPCProject, Serge Guelton nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE(paddle-dev): We introduce third-party library Gast as unified AST +# representation. See https://github.com/serge-sans-paille/gast for details. + +import sys as _sys +import ast as _ast +from ast import boolop, cmpop, excepthandler, expr, expr_context, operator +from ast import slice, stmt, unaryop, mod, AST +from ast import iter_child_nodes, walk + +try: + from ast import TypeIgnore +except ImportError: + + class TypeIgnore(AST): + pass + + +def _make_node(Name, Fields, Attributes, Bases): + def create_node(self, *args, **kwargs): + nbparam = len(args) + len(kwargs) + assert nbparam in (0, len(Fields)), \ + "Bad argument number for {}: {}, expecting {}".\ + format(Name, nbparam, len(Fields)) + self._fields = Fields + self._attributes = Attributes + for argname, argval in zip(self._fields, args): + setattr(self, argname, argval) + for argname, argval in kwargs.items(): + assert argname in Fields, \ + "Invalid Keyword argument for {}: {}".format(Name, argname) + setattr(self, argname, argval) + + setattr(_sys.modules[__name__], Name, + type(Name, Bases, {'__init__': create_node})) + + +_nodes = ( + # mod + ('Module', (('body', 'type_ignores'), (), (mod, ))), + ('Interactive', (('body', ), (), (mod, ))), + ('Expression', (('body', ), (), (mod, ))), + ('FunctionType', (('argtypes', 'returns'), (), (mod, ))), + ('Suite', (('body', ), (), (mod, ))), + + # stmt + ('FunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', + 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncFunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', + 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('ClassDef', (( + 'name', + 'bases', + 'keywords', + 'body', + 'decorator_list', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Return', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Delete', (('targets', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Assign', (( + 'targets', + 'value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AugAssign', (( + 'target', + 'op', + 'value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AnnAssign', (( + 'target', + 'annotation', + 'value', + 'simple', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Print', (( + 'dest', + 'values', + 'nl', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('For', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncFor', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('While', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('If', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('With', (('items', 'body', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('AsyncWith', (('items', 'body', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Raise', (( + 'exc', + 'cause', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Try', (( + 'body', + 'handlers', + 'orelse', + 'finalbody', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Assert', (( + 'test', + 'msg', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Import', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('ImportFrom', (( + 'module', + 'names', + 'level', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Exec', (( + 'body', + 'globals', + 'locals', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Global', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Nonlocal', (('names', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Expr', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Pass', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Break', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + ('Continue', ((), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (stmt, ))), + + # expr + ('BoolOp', (( + 'op', + 'values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('BinOp', (( + 'left', + 'op', + 'right', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('UnaryOp', (( + 'op', + 'operand', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Lambda', (( + 'args', + 'body', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('IfExp', (( + 'test', + 'body', + 'orelse', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Dict', (( + 'keys', + 'values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Set', (('elts', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('ListComp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('SetComp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('DictComp', (( + 'key', + 'value', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('GeneratorExp', (( + 'elt', + 'generators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Await', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Yield', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('YieldFrom', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Compare', (( + 'left', + 'ops', + 'comparators', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Call', (( + 'func', + 'args', + 'keywords', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Repr', (('value', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('FormattedValue', (( + 'value', + 'conversion', + 'format_spec', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('JoinedStr', (('values', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Constant', (('value', 'kind'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Attribute', (( + 'value', + 'attr', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Subscript', (( + 'value', + 'slice', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Starred', (( + 'value', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Name', (('id', 'ctx', 'annotation', 'type_comment'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('List', (( + 'elts', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + ('Tuple', (( + 'elts', + 'ctx', ), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (expr, ))), + + # expr_context + ('Load', ((), (), (expr_context, ))), + ('Store', ((), (), (expr_context, ))), + ('Del', ((), (), (expr_context, ))), + ('AugLoad', ((), (), (expr_context, ))), + ('AugStore', ((), (), (expr_context, ))), + ('Param', ((), (), (expr_context, ))), + + # slice + ('Slice', (('lower', 'upper', 'step'), ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', ), (slice, ))), + + # boolop + ('And', ((), (), (boolop, ))), + ('Or', ((), (), (boolop, ))), + + # operator + ('Add', ((), (), (operator, ))), + ('Sub', ((), (), (operator, ))), + ('Mult', ((), (), (operator, ))), + ('MatMult', ((), (), (operator, ))), + ('Div', ((), (), (operator, ))), + ('Mod', ((), (), (operator, ))), + ('Pow', ((), (), (operator, ))), + ('LShift', ((), (), (operator, ))), + ('RShift', ((), (), (operator, ))), + ('BitOr', ((), (), (operator, ))), + ('BitXor', ((), (), (operator, ))), + ('BitAnd', ((), (), (operator, ))), + ('FloorDiv', ((), (), (operator, ))), + + # unaryop + ('Invert', ((), (), ( + unaryop, + AST, ))), + ('Not', ((), (), ( + unaryop, + AST, ))), + ('UAdd', ((), (), ( + unaryop, + AST, ))), + ('USub', ((), (), ( + unaryop, + AST, ))), + + # cmpop + ('Eq', ((), (), (cmpop, ))), + ('NotEq', ((), (), (cmpop, ))), + ('Lt', ((), (), (cmpop, ))), + ('LtE', ((), (), (cmpop, ))), + ('Gt', ((), (), (cmpop, ))), + ('GtE', ((), (), (cmpop, ))), + ('Is', ((), (), (cmpop, ))), + ('IsNot', ((), (), (cmpop, ))), + ('In', ((), (), (cmpop, ))), + ('NotIn', ((), (), (cmpop, ))), + + # comprehension + ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST, ))), + + # excepthandler + ('ExceptHandler', (('type', 'name', 'body'), + ('lineno', 'col_offset', 'end_lineno', + 'end_col_offset'), (excepthandler, ))), + + # arguments + ('arguments', (('args', 'posonlyargs', 'vararg', 'kwonlyargs', + 'kw_defaults', 'kwarg', 'defaults'), (), (AST, ))), + + # keyword + ('keyword', + (('arg', 'value'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), (AST, ))), + + # alias + ('alias', (('name', 'asname'), (), (AST, ))), + + # withitem + ('withitem', (('context_expr', 'optional_vars'), (), (AST, ))), + + # type_ignore + ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore, ))), ) + +for name, descr in _nodes: + _make_node(name, *descr) + +py_version = _sys.version_info.major +if py_version != 3: + raise RuntimeError( + 'Required Python version >= 3, but received Python version == {}'. + format(py_version)) + +from .ast3 import ast_to_gast, gast_to_ast + + +def parse(*args, **kwargs): + return ast_to_gast(_ast.parse(*args, **kwargs)) + + +def literal_eval(node_or_string): + if isinstance(node_or_string, AST): + node_or_string = gast_to_ast(node_or_string) + return _ast.literal_eval(node_or_string) + + +def get_docstring(node, clean=True): + if not isinstance(node, (FunctionDef, ClassDef, Module)): + raise TypeError("%r can't have docstrings" % node.__class__.__name__) + if node.body and isinstance(node.body[0], Expr) and \ + isinstance(node.body[0].value, Constant): + if clean: + import inspect + holder = node.body[0].value + return inspect.cleandoc(getattr(holder, holder._fields[0])) + return node.body[0].value.s + + +# the following are directly imported from python3.8's Lib/ast.py # + + +def copy_location(new_node, old_node): + """ + Copy source location (`lineno`, `col_offset`, `end_lineno`, and + `end_col_offset` attributes) from *old_node* to *new_node* if possible, + and return *new_node*. + """ + for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': + if attr in old_node._attributes and attr in new_node._attributes \ + and hasattr(old_node, attr): + setattr(new_node, attr, getattr(old_node, attr)) + return new_node + + +def fix_missing_locations(node): + """ + When you compile a node tree with compile(), the compiler expects lineno + and col_offset attributes for every node that supports them. This is + rather tedious to fill in for generated nodes, so this helper adds these + attributes recursively where not already set, by setting them to the values + of the parent node. It works recursively starting at *node*. + """ + + def _fix(node, lineno, col_offset, end_lineno, end_col_offset): + if 'lineno' in node._attributes: + if not hasattr(node, 'lineno'): + node.lineno = lineno + else: + lineno = node.lineno + if 'end_lineno' in node._attributes: + if not hasattr(node, 'end_lineno'): + node.end_lineno = end_lineno + else: + end_lineno = node.end_lineno + if 'col_offset' in node._attributes: + if not hasattr(node, 'col_offset'): + node.col_offset = col_offset + else: + col_offset = node.col_offset + if 'end_col_offset' in node._attributes: + if not hasattr(node, 'end_col_offset'): + node.end_col_offset = end_col_offset + else: + end_col_offset = node.end_col_offset + for child in iter_child_nodes(node): + _fix(child, lineno, col_offset, end_lineno, end_col_offset) + + _fix(node, 1, 0, 1, 0) + return node + + +def increment_lineno(node, n=1): + """ + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. + """ + for child in walk(node): + if 'lineno' in child._attributes: + child.lineno = (getattr(child, 'lineno', 0) or 0) + n + if 'end_lineno' in child._attributes: + child.end_lineno = (getattr(child, 'end_lineno', 0) or 0) + n + return node diff --git a/python/requirements.txt b/python/requirements.txt index e9da2aa24d6cb2b352bbc4d855f531401dbdfd46..4232700761581cb9486a866254786eed3796dcfd 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -2,8 +2,6 @@ requests>=2.20.0 numpy>=1.13 ; python_version>="3.5" and platform_system != "Windows" numpy>=1.13, <=1.19.3 ; python_version>="3.5" and platform_system == "Windows" protobuf>=3.1.0 -gast>=0.3.3, <=0.4.0 ; platform_system != "Windows" -gast==0.3.3 ; platform_system == "Windows" Pillow six decorator diff --git a/python/setup.py.in b/python/setup.py.in index 64b5dd4cd21063c7db08a51b65552d0d587f5953..f4535fe157d69a7105a8769e9f67166735fb9e1f 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -139,6 +139,7 @@ write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/f packages=['paddle', 'paddle.libs', 'paddle.utils', + 'paddle.utils.gast', 'paddle.utils.cpp_extension', 'paddle.dataset', 'paddle.reader',