diff --git a/auto_py2to3/libs/lib2to3/btm_utils.py b/auto_py2to3/libs/lib2to3/btm_utils.py index c78fea1c2c3f040e29b33945218b048a932262cd..dc16d19c188766ca481d8a669198669c48972f8e 100644 --- a/auto_py2to3/libs/lib2to3/btm_utils.py +++ b/auto_py2to3/libs/lib2to3/btm_utils.py @@ -19,8 +19,8 @@ class MinNode(object): pattern tree during the conversion to sets of leaf-to-root subpatterns""" - def __init__(self, type=None, name=None): - self.type = type + def __init__(self, _type=None, name=None): + self.type = _type self.name = name self.children = [] self.leaf = False @@ -220,9 +220,7 @@ def reduce_tree(node, parent=None): # reduce to a single occurrence i.e. do nothing pass else: - # TODO: handle {min, max} repeaters raise NotImplementedError - pass # add children if details_node and new_node is not None: diff --git a/auto_py2to3/libs/lib2to3/fixer_base.py b/auto_py2to3/libs/lib2to3/fixer_base.py index 7e14be67ac5b48dcd8409e7e00758091a548565e..79f69ee39c4ee481876361e146a4fddb6d13307c 100644 --- a/auto_py2to3/libs/lib2to3/fixer_base.py +++ b/auto_py2to3/libs/lib2to3/fixer_base.py @@ -7,6 +7,8 @@ import itertools # Local imports +from abc import ABC + from .patcomp import PatternCompiler from . import pygram from .fixer_util import does_tree_import @@ -42,7 +44,7 @@ class BaseFix(object): # manually # Shortcut for access to Python grammar symbols - syms = pygram.python_symbols + sym_s = pygram.python_symbols def __init__(self, options, log): """Initializer. Subclass may override. @@ -52,6 +54,7 @@ class BaseFix(object): that could be used to customize the fixer through the command line. log: a list to append warnings and other messages to. """ + self.first_log = False self.options = options self.log = log self.compile_pattern() @@ -115,7 +118,6 @@ class BaseFix(object): def log_message(self, message): if self.first_log: - self.first_log = False self.log.append("### In file %s ###" % self.filename) self.log.append(message) @@ -166,15 +168,18 @@ class BaseFix(object): pass -class ConditionalFix(BaseFix): +class ConditionalFix(BaseFix, ABC): """ Base class for fixers which not execute if an import is found. """ # This is the name of the import which, if found, will cause the test to be skipped skip_on = None + def __init__(self, options, log): + super().__init__(options, log) + self._should_skip = None + def start_tree(self, *args): super(ConditionalFix, self).start_tree(*args) - self._should_skip = None def should_skip(self, node): if self._should_skip is not None: diff --git a/auto_py2to3/libs/lib2to3/fixer_util.py b/auto_py2to3/libs/lib2to3/fixer_util.py index 29d97ce4c950b0f01dbe52fd1bba928a63b4ed10..76d7e64da2b1532ee9ddd22f048ac166b39fd979 100644 --- a/auto_py2to3/libs/lib2to3/fixer_util.py +++ b/auto_py2to3/libs/lib2to3/fixer_util.py @@ -127,11 +127,6 @@ def ListComp(xp, fp, it, test=None): def FromImport(package_name, name_leafs): """ Return an import statement in the form: from package import name_leafs""" - # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') - # assert package_name == '.' or '.' not in package_name, "FromImport has "\ - # "not been tested with dotted package names -- use at your own "\ - # "peril!" - for leaf in name_leafs: # Pull the leaves out of their old tree leaf.remove() @@ -221,31 +216,10 @@ def attr_chain(obj, attr): Yields: Each successive object in the chain. """ - next = getattr(obj, attr) - while next: - yield next - next = getattr(next, attr) - - -p0 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ -p1 = """ -power< - ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | - 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) - trailer< '(' node=any ')' > - any* -> -""" -p2 = """ -power< - ( 'sorted' | 'enumerate' ) - trailer< '(' arglist ')' > - any* -> -""" -pats_built = False + _next = getattr(obj, attr) + while _next: + yield _next + _next = getattr(_next, attr) def in_special_context(node): @@ -254,7 +228,25 @@ def in_special_context(node): or an iterator). See test_map_nochange in test_fixers.py for some examples and tests. """ - global p0, p1, p2, pats_built + p0 = """for_stmt< 'for' any 'in' node=any ':' any* > + | comp_for< 'for' any 'in' node=any any* > + """ + p1 = """ + power< + ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | + 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) + trailer< '(' node=any ')' > + any* + > + """ + p2 = """ + power< + ( 'sorted' | 'enumerate' ) + trailer< '(' arglist ')' > + any* + > + """ + pats_built = False if not pats_built: p0 = patcomp.compile_pattern(p0) p1 = patcomp.compile_pattern(p1) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_apply.py b/auto_py2to3/libs/lib2to3/fixes/fix_apply.py index d3b5e94819c82d07a6ac3b1a3e69540959025427..4a6052e56bc2073797e5f140b02b08a2d5e25f01 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_apply.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_apply.py @@ -30,7 +30,7 @@ class FixApply(fixer_base.BaseFix): """ def transform(self, node, results): - syms = self.syms + sym_s = self.syms assert results func = results["func"] args = results["args"] @@ -48,8 +48,8 @@ class FixApply(fixer_base.BaseFix): return # Make no change. prefix = node.prefix func = func.clone() - if (func.type not in (token.NAME, syms.atom) and - (func.type != syms.power or + if (func.type not in (token.NAME, sym_s.atom) and + (func.type != sym_s.power or func.children[-2].type == token.DOUBLESTAR)): # Need to parenthesize func = parenthesize(func) @@ -64,8 +64,5 @@ class FixApply(fixer_base.BaseFix): l_newargs.extend([Comma(), pytree.Leaf(token.DOUBLESTAR, "**"), kwds]) - l_newargs[-2].prefix = " " # that's the ** token - # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t) - # can be translated into f(x, y, *t) instead of f(*(x, y) + t) - # new = pytree.Node(syms.power, (func, ArgList(l_newargs))) + l_newargs[-2].prefix = " " return Call(func, l_newargs, prefix=prefix) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_dict.py b/auto_py2to3/libs/lib2to3/fixes/fix_dict.py index c029bfccee98b4c885bf6ac0814013f0b0720fe4..776b7debafafd3054912f13bc6be746fd9b566dc 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_dict.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_dict.py @@ -54,7 +54,7 @@ class FixDict(fixer_base.BaseFix): head = results["head"] method = results["method"][0] # Extract node for method name tail = results["tail"] - syms = self.syms + sym_s = self.syms method_name = method.value isiter = method_name.startswith("iter") isview = method_name.startswith("view") @@ -64,17 +64,17 @@ class FixDict(fixer_base.BaseFix): head = [n.clone() for n in head] tail = [n.clone() for n in tail] special = not tail and self.in_special_context(node, isiter) - args = head + [pytree.Node(syms.trailer, + args = head + [pytree.Node(sym_s.trailer, [Dot(), Name(method_name, prefix=method.prefix)]), results["parens"].clone()] - new = pytree.Node(syms.power, args) + new = pytree.Node(sym_s.power, args) if not (special or isview): new.prefix = "" new = Call(Name("iter" if isiter else "list"), [new]) if tail: - new = pytree.Node(syms.power, [new] + tail) + new = pytree.Node(sym_s.power, [new] + tail) new.prefix = node.prefix return new diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_except.py b/auto_py2to3/libs/lib2to3/fixes/fix_except.py index 6068755197177600b7e3640fedb5f5d51aeb5107..0fd50356f219d0ade318d5462545704c5518c80f 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_except.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_except.py @@ -25,12 +25,12 @@ The following cases will be converted: from .. import pytree from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms +from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, sym_s def find_excepts(nodes): for i, n in enumerate(nodes): - if n.type == syms.except_clause: + if n.type == sym_s.except_clause: if n.children[0].value == 'except': yield (n, nodes[i + 2]) @@ -47,7 +47,7 @@ class FixExcept(fixer_base.BaseFix): """ def transform(self, node, results): - syms = self.syms + sym_s = self.syms tail = [n.clone() for n in results["tail"]] @@ -70,6 +70,7 @@ class FixExcept(fixer_base.BaseFix): # and indents # TODO(cwinter) suite-cleanup suite_stmts = e_suite.children + i = 0 for i, stmt in enumerate(suite_stmts): if isinstance(stmt, pytree.Node): break @@ -81,7 +82,6 @@ class FixExcept(fixer_base.BaseFix): else: assign = Assign(target, new_N) - # TODO(cwinter) stopgap until children becomes a smart list for child in reversed(suite_stmts[:i]): e_suite.insert_child(0, child) e_suite.insert_child(i, assign) @@ -90,6 +90,5 @@ class FixExcept(fixer_base.BaseFix): # not so much. N.prefix = " " - # TODO(cwinter) fix this when children becomes a smart list children = [c.clone() for c in node.children[:3]] + try_cleanup + tail return pytree.Node(node.type, children) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_exec.py b/auto_py2to3/libs/lib2to3/fixes/fix_exec.py index ab921ee80cdf366532f027b2549e5bcba55a4fd5..2633e6d89175a18bd380618a77c53b19091156ab 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_exec.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_exec.py @@ -25,7 +25,7 @@ class FixExec(fixer_base.BaseFix): def transform(self, node, results): assert results - syms = self.syms + sym_s = self.syms a = results["a"] b = results.get("b") c = results.get("c") diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_execfile.py b/auto_py2to3/libs/lib2to3/fixes/fix_execfile.py index b6c786fd4e8b6a141ab3619ba2b2576db158fcd4..8bb9e1ea3bc591bf7e02a74afe92d90f368400ea 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_execfile.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_execfile.py @@ -24,8 +24,8 @@ class FixExecfile(fixer_base.BaseFix): def transform(self, node, results): assert results filename = results["filename"] - globals = results.get("globals") - locals = results.get("locals") + tmp_globals = results.get("globals") + tmp_locals = results.get("locals") # Copy over the prefix from the right parentheses end of the execfile # call. @@ -46,8 +46,8 @@ class FixExecfile(fixer_base.BaseFix): compile_call = Call(Name("compile"), compile_args, "") # Finally, replace the execfile call with an exec call. args = [compile_call] - if globals is not None: - args.extend([Comma(), globals.clone()]) - if locals is not None: - args.extend([Comma(), locals.clone()]) + if tmp_globals is not None: + args.extend([Comma(), tmp_globals.clone()]) + if tmp_locals is not None: + args.extend([Comma(), tmp_locals.clone()]) return Call(Name("exec"), args, prefix=node.prefix) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_has_key.py b/auto_py2to3/libs/lib2to3/fixes/fix_has_key.py index 439708c9923404312dfabb964615062ea32c0aea..d42dbdf91e8a1e7f04a84ff76957937c639bb984 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_has_key.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_has_key.py @@ -70,8 +70,8 @@ class FixHasKey(fixer_base.BaseFix): def transform(self, node, results): assert results - syms = self.syms - if (node.parent.type == syms.not_test and + sym_s = self.syms + if (node.parent.type == sym_s.not_test and self.pattern.match(node.parent)): # Don't transform a node matching the first alternative of the # pattern when its parent matches the second alternative @@ -84,26 +84,26 @@ class FixHasKey(fixer_base.BaseFix): after = results.get("after") if after: after = [n.clone() for n in after] - if arg.type in (syms.comparison, syms.not_test, syms.and_test, - syms.or_test, syms.test, syms.lambdef, syms.argument): + if arg.type in (sym_s.comparison, sym_s.not_test, sym_s.and_test, + sym_s.or_test, sym_s.test, sym_s.lambdef, sym_s.argument): arg = parenthesize(arg) if len(before) == 1: before = before[0] else: - before = pytree.Node(syms.power, before) + before = pytree.Node(sym_s.power, before) before.prefix = " " n_op = Name("in", prefix=" ") if negation: n_not = Name("not", prefix=" ") - n_op = pytree.Node(syms.comp_op, (n_not, n_op)) - new = pytree.Node(syms.comparison, (arg, n_op, before)) + n_op = pytree.Node(sym_s.comp_op, (n_not, n_op)) + new = pytree.Node(sym_s.comparison, (arg, n_op, before)) if after: new = parenthesize(new) - new = pytree.Node(syms.power, (new,) + tuple(after)) - if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr, - syms.and_expr, syms.shift_expr, - syms.arith_expr, syms.term, - syms.factor, syms.power): + new = pytree.Node(sym_s.power, (new,) + tuple(after)) + if node.parent.type in (sym_s.comparison, sym_s.expr, sym_s.xor_expr, + sym_s.and_expr, sym_s.shift_expr, + sym_s.arith_expr, sym_s.term, + sym_s.factor, sym_s.power): new = parenthesize(new) new.prefix = prefix return new diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_raise.py b/auto_py2to3/libs/lib2to3/fixes/fix_raise.py index d870d14a202fbee2374cdb5a1e558483dd1f927e..e4d1e6ea4bdcef8b70edf5ecdd7cfb4fab837a19 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_raise.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_raise.py @@ -36,7 +36,7 @@ class FixRaise(fixer_base.BaseFix): """ def transform(self, node, results): - syms = self.syms + sym_s = self.syms exc = results["exc"].clone() if exc.type == token.STRING: @@ -59,7 +59,7 @@ class FixRaise(fixer_base.BaseFix): if "val" not in results: # One-argument raise - new = pytree.Node(syms.raise_stmt, [Name("raise"), exc]) + new = pytree.Node(sym_s.raise_stmt, [Name("raise"), exc]) new.prefix = node.prefix return new @@ -81,10 +81,10 @@ class FixRaise(fixer_base.BaseFix): if val.type != token.NAME or val.value != "None": e = Call(exc, args) with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb) + new = pytree.Node(sym_s.simple_stmt, [Name("raise")] + with_tb) new.prefix = node.prefix return new else: - return pytree.Node(syms.raise_stmt, + return pytree.Node(sym_s.raise_stmt, [Name("raise"), Call(exc, args)], prefix=node.prefix) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_throw.py b/auto_py2to3/libs/lib2to3/fixes/fix_throw.py index b340b4964f57a858f0f6940967c8671f7f233d28..ec06925bc5e0ae4dc6b5e9733818c4cc71460a40 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_throw.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_throw.py @@ -25,7 +25,7 @@ class FixThrow(fixer_base.BaseFix): """ def transform(self, node, results): - syms = self.syms + sym_s = self.syms exc = results["exc"].clone() if exc.type is token.STRING: @@ -52,6 +52,6 @@ class FixThrow(fixer_base.BaseFix): e = Call(exc, args) with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - throw_args.replace(pytree.Node(syms.power, with_tb)) + throw_args.replace(pytree.Node(sym_s.power, with_tb)) else: throw_args.replace(Call(exc, args)) diff --git a/auto_py2to3/libs/lib2to3/fixes/fix_tuple_params.py b/auto_py2to3/libs/lib2to3/fixes/fix_tuple_params.py index 1319747e1de42be03f53d2348b44f869b4592652..6b670b3fdb8f4d9fcf035e5acab77c11ea1ccd38 100644 --- a/auto_py2to3/libs/lib2to3/fixes/fix_tuple_params.py +++ b/auto_py2to3/libs/lib2to3/fixes/fix_tuple_params.py @@ -94,7 +94,6 @@ class FixTupleParams(fixer_base.BaseFix): for line in new_lines: line.parent = suite[0] - # TODO(cwinter) suite-cleanup after = start if start == 0: new_lines[0].prefix = " "