[Python-checkins] r67811 - in python/branches/release30-maint/Lib/lib2to3: fixer_base.py fixer_util.py fixes/fix_apply.py fixes/fix_has_key.py fixes/fix_imports.py fixes/fix_imports2.py fixes/fix_intern.py fixes/fix_isinstance.py fixes/fix_long.py fixes/fix_reduce.py fixes/fix_repr.py fixes/fix_urllib.py fixes/fix_xrange.py main.py pygram.py pytree.py refactor.py tests/benchmark.py tests/data/infinite_recursion.py tests/test_fixers.py tests/test_util.py

Tue Dec 16 05:13:51 CET 2008

Author: benjamin.peterson
Date: Tue Dec 16 05:13:50 2008
New Revision: 67811
Log:
Merged revisions 67810 via svnmerge from 
svn+ssh://pythondev@svn.python.org/python/branches/py3k
................
 r67810 | benjamin.peterson | 2008年12月15日 21:57:54 -0600 (2008年12月15日) | 118 lines
 
 Merged revisions 67806 via svnmerge from 
 svn+ssh://pythondev@svn.python.org/python/trunk
 
 ................
 r67806 | benjamin.peterson | 2008年12月15日 21:35:28 -0600 (2008年12月15日) | 111 lines
 
 Merged revisions 67427,67431,67433,67435,67630,67652,67656-67657,67674-67675,67678-67679,67705-67706,67716,67723,67765-67771,67774,67776,67778 via svnmerge from 
 svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3
 
 ........
 r67427 | benjamin.peterson | 2008年11月28日 16:07:41 -0600 (2008年11月28日) | 1 line
 
 fix spelling in comment
 ........
 r67431 | benjamin.peterson | 2008年11月28日 17:14:08 -0600 (2008年11月28日) | 1 line
 
 add a scripts directory; move things to it
 ........
 r67433 | benjamin.peterson | 2008年11月28日 17:18:48 -0600 (2008年11月28日) | 1 line
 
 run svneol.py
 ........
 r67435 | benjamin.peterson | 2008年11月28日 17:25:03 -0600 (2008年11月28日) | 1 line
 
 rename pre/post_order_mapping to pre/post_order_heads
 ........
 r67630 | alexandre.vassalotti | 2008年12月06日 21:51:56 -0600 (2008年12月06日) | 2 lines
 
 Fix typo in the urllib2.HTTPDigestAuthHandler fixer.
 ........
 r67652 | armin.ronacher | 2008年12月07日 15:39:43 -0600 (2008年12月07日) | 5 lines
 
 Added a fixer that cleans up a tuple argument to isinstance after the tokens
 in it were fixed. This is mainly used to remove double occurrences of
 tokens as a leftover of the long -> int / unicode -> str conversion.
 ........
 r67656 | armin.ronacher | 2008年12月07日 16:54:16 -0600 (2008年12月07日) | 3 lines
 
 Added missing copyright fo 2to3 fix_isinstance.
 ........
 r67657 | armin.ronacher | 2008年12月07日 18:29:35 -0600 (2008年12月07日) | 3 lines
 
 2to3: intern and reduce fixes now add the imports if missing. Because that is a common task the fixer_util module now has a function "touch_import" that adds imports if missing.
 ........
 r67674 | benjamin.peterson | 2008年12月08日 19:58:11 -0600 (2008年12月08日) | 1 line
 
 copy permission bits when making backup files #4602
 ........
 r67675 | benjamin.peterson | 2008年12月08日 19:59:11 -0600 (2008年12月08日) | 1 line
 
 add forgotten import
 ........
 r67678 | benjamin.peterson | 2008年12月08日 20:08:30 -0600 (2008年12月08日) | 1 line
 
 fix #4602 for real
 ........
 r67679 | armin.ronacher | 2008年12月09日 00:54:03 -0600 (2008年12月09日) | 3 lines
 
 Removed redudant code from the 2to3 long fixer. This fixes #4590.
 ........
 r67705 | benjamin.peterson | 2008年12月11日 13:04:08 -0600 (2008年12月11日) | 1 line
 
 put trailers after a range call after the list()
 ........
 r67706 | benjamin.peterson | 2008年12月11日 13:17:57 -0600 (2008年12月11日) | 1 line
 
 add html related modules to the fix_imports mapping
 ........
 r67716 | benjamin.peterson | 2008年12月11日 22:16:47 -0600 (2008年12月11日) | 1 line
 
 consolidate tests
 ........
 r67723 | benjamin.peterson | 2008年12月12日 19:49:31 -0600 (2008年12月12日) | 1 line
 
 fix name
 ........
 r67765 | benjamin.peterson | 2008年12月14日 14:05:05 -0600 (2008年12月14日) | 1 line
 
 run fix_isinstance after fix_long and fix_unicode
 ........
 r67766 | benjamin.peterson | 2008年12月14日 14:13:05 -0600 (2008年12月14日) | 1 line
 
 use run_order instead of order
 ........
 r67767 | benjamin.peterson | 2008年12月14日 14:28:12 -0600 (2008年12月14日) | 1 line
 
 don't retain parenthesis if there is only one item left
 ........
 r67768 | benjamin.peterson | 2008年12月14日 14:32:30 -0600 (2008年12月14日) | 1 line
 
 use insert_child()
 ........
 r67769 | benjamin.peterson | 2008年12月14日 14:59:10 -0600 (2008年12月14日) | 1 line
 
 parenthesize doesn't belong in pygram or FixerBase
 ........
 r67770 | alexandre.vassalotti | 2008年12月14日 15:15:36 -0600 (2008年12月14日) | 2 lines
 
 Fix typo: html.paser -> html.parser.
 ........
 r67771 | benjamin.peterson | 2008年12月14日 15:22:09 -0600 (2008年12月14日) | 1 line
 
 altering .children needs to call changed()
 ........
 r67774 | benjamin.peterson | 2008年12月14日 15:55:38 -0600 (2008年12月14日) | 1 line
 
 employ an evil hack to fix multiple names in the same import statement
 ........
 r67776 | benjamin.peterson | 2008年12月14日 16:22:38 -0600 (2008年12月14日) | 1 line
 
 make a common mixin class for Test_imports and friends
 ........
 r67778 | alexandre.vassalotti | 2008年12月14日 17:48:20 -0600 (2008年12月14日) | 2 lines
 
 Make fix_imports refactor multiple imports as.
 ........
 ................
................
Added:
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_isinstance.py
 - copied unchanged from r67810, /python/branches/py3k/Lib/lib2to3/fixes/fix_isinstance.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_reduce.py
 - copied unchanged from r67810, /python/branches/py3k/Lib/lib2to3/fixes/fix_reduce.py
Removed:
 python/branches/release30-maint/Lib/lib2to3/tests/benchmark.py
Modified:
 python/branches/release30-maint/Lib/lib2to3/fixer_base.py
 python/branches/release30-maint/Lib/lib2to3/fixer_util.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_apply.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_has_key.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports2.py (contents, props changed)
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_intern.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_long.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_repr.py
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_urllib.py (contents, props changed)
 python/branches/release30-maint/Lib/lib2to3/fixes/fix_xrange.py
 python/branches/release30-maint/Lib/lib2to3/main.py
 python/branches/release30-maint/Lib/lib2to3/pygram.py
 python/branches/release30-maint/Lib/lib2to3/pytree.py
 python/branches/release30-maint/Lib/lib2to3/refactor.py
 python/branches/release30-maint/Lib/lib2to3/tests/data/infinite_recursion.py (props changed)
 python/branches/release30-maint/Lib/lib2to3/tests/test_fixers.py
 python/branches/release30-maint/Lib/lib2to3/tests/test_util.py
Modified: python/branches/release30-maint/Lib/lib2to3/fixer_base.py
==============================================================================

--- python/branches/release30-maint/Lib/lib2to3/fixer_base.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixer_base.py	Tue Dec 16 05:13:50 2008
@@ -94,10 +94,6 @@
 """
 raise NotImplementedError()
 
- def parenthesize(self, node):
- """Wrapper around pygram.parenthesize()."""
- return pygram.parenthesize(node)
-
 def new_name(self, template="xxx_todo_changeme"):
 """Return a string suitable for use as an identifier
 
Modified: python/branches/release30-maint/Lib/lib2to3/fixer_util.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixer_util.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixer_util.py	Tue Dec 16 05:13:50 2008
@@ -158,6 +158,9 @@
 ### Misc
 ###########################################################
 
+def parenthesize(node):
+ return Node(syms.atom, [LParen(), node, RParen()])
+
 
 consuming_calls = set(["sorted", "list", "set", "any", "all", "tuple", "sum",
 "min", "max"])
@@ -232,20 +235,77 @@
 suite.parent = parent
 return suite
 
-def does_tree_import(package, name, node):
- """ Returns true if name is imported from package at the
- top level of the tree which node belongs to.
- To cover the case of an import like 'import foo', use
- Null for the package and 'foo' for the name. """
+def find_root(node):
+ """Find the top level namespace."""
 # Scamper up to the top level namespace
 while node.type != syms.file_input:
 assert node.parent, "Tree is insane! root found before "\
 "file_input node was found."
 node = node.parent
+ return node
 
- binding = find_binding(name, node, package)
+def does_tree_import(package, name, node):
+ """ Returns true if name is imported from package at the
+ top level of the tree which node belongs to.
+ To cover the case of an import like 'import foo', use
+ None for the package and 'foo' for the name. """
+ binding = find_binding(name, find_root(node), package)
 return bool(binding)
 
+def is_import(node):
+ """Returns true if the node is an import statement."""
+ return node.type in (syms.import_name, syms.import_from)
+
+def touch_import(package, name, node):
+ """ Works like `does_tree_import` but adds an import statement
+ if it was not imported. """
+ def is_import_stmt(node):
+ return node.type == syms.simple_stmt and node.children and \
+ is_import(node.children[0])
+
+ root = find_root(node)
+
+ if does_tree_import(package, name, root):
+ return
+
+ add_newline_before = False
+
+ # figure out where to insert the new import. First try to find
+ # the first import and then skip to the last one.
+ insert_pos = offset = 0
+ for idx, node in enumerate(root.children):
+ if not is_import_stmt(node):
+ continue
+ for offset, node2 in enumerate(root.children[idx:]):
+ if not is_import_stmt(node2):
+ break
+ insert_pos = idx + offset
+ break
+
+ # if there are no imports where we can insert, find the docstring.
+ # if that also fails, we stick to the beginning of the file
+ if insert_pos == 0:
+ for idx, node in enumerate(root.children):
+ if node.type == syms.simple_stmt and node.children and \
+ node.children[0].type == token.STRING:
+ insert_pos = idx + 1
+ add_newline_before
+ break
+
+ if package is None:
+ import_ = Node(syms.import_name, [
+ Leaf(token.NAME, 'import'),
+ Leaf(token.NAME, name, prefix=' ')
+ ])
+ else:
+ import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')])
+
+ children = [import_, Newline()]
+ if add_newline_before:
+ children.insert(0, Newline())
+ root.insert_child(insert_pos, Node(syms.simple_stmt, children))
+
+
 _def_syms = set([syms.classdef, syms.funcdef])
 def find_binding(name, node, package=None):
 """ Returns the node which binds variable name, otherwise None.
@@ -285,7 +345,7 @@
 if ret:
 if not package:
 return ret
- if ret.type in (syms.import_name, syms.import_from):
+ if is_import(ret):
 return ret
 return None
 
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_apply.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_apply.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_apply.py	Tue Dec 16 05:13:50 2008
@@ -9,7 +9,7 @@
 from .. import pytree
 from ..pgen2 import token
 from .. import fixer_base
-from ..fixer_util import Call, Comma
+from ..fixer_util import Call, Comma, parenthesize
 
 class FixApply(fixer_base.BaseFix):
 
@@ -39,7 +39,7 @@
 (func.type != syms.power or
 func.children[-2].type == token.DOUBLESTAR)):
 # Need to parenthesize
- func = self.parenthesize(func)
+ func = parenthesize(func)
 func.set_prefix("")
 args = args.clone()
 args.set_prefix("")
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_has_key.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_has_key.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_has_key.py	Tue Dec 16 05:13:50 2008
@@ -33,7 +33,7 @@
 from .. import pytree
 from ..pgen2 import token
 from .. import fixer_base
-from ..fixer_util import Name
+from ..fixer_util import Name, parenthesize
 
 
 class FixHasKey(fixer_base.BaseFix):
@@ -86,7 +86,7 @@
 after = [n.clone() for n in after]
 if arg.type in (syms.comparison, syms.not_test, syms.and_test,
 syms.or_test, syms.test, syms.lambdef, syms.argument):
- arg = self.parenthesize(arg)
+ arg = parenthesize(arg)
 if len(before) == 1:
 before = before[0]
 else:
@@ -98,12 +98,12 @@
 n_op = pytree.Node(syms.comp_op, (n_not, n_op))
 new = pytree.Node(syms.comparison, (arg, n_op, before))
 if after:
- new = self.parenthesize(new)
+ new = parenthesize(new)
 new = pytree.Node(syms.power, (new,) + tuple(after))
 if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr,
 syms.and_expr, syms.shift_expr,
 syms.arith_expr, syms.term,
 syms.factor, syms.power):
- new = self.parenthesize(new)
+ new = parenthesize(new)
 new.set_prefix(prefix)
 return new
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports.py	Tue Dec 16 05:13:50 2008
@@ -42,6 +42,8 @@
 'DocXMLRPCServer': 'xmlrpc.server',
 'SimpleXMLRPCServer': 'xmlrpc.server',
 'httplib': 'http.client',
+ 'htmlentitydefs' : 'html.entities',
+ 'HTMLParser' : 'html.parser',
 'Cookie': 'http.cookies',
 'cookielib': 'http.cookiejar',
 'BaseHTTPServer': 'http.server',
@@ -64,16 +66,17 @@
 mod_list = ' | '.join(["module_name='%s'" % key for key in mapping])
 bare_names = alternates(mapping.keys())
 
- yield """name_import=import_name< 'import' ((%s)
- | dotted_as_names< any* (%s) any* >) >
+ yield """name_import=import_name< 'import' ((%s) |
+ multiple_imports=dotted_as_names< any* (%s) any* >) >
 """ % (mod_list, mod_list)
 yield """import_from< 'from' (%s) 'import' ['(']
 ( any | import_as_name< any 'as' any > |
 import_as_names< any* >) [')'] >
 """ % mod_list
- yield """import_name< 'import'
- dotted_as_name< (%s) 'as' any > >
- """ % mod_list
+ yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > |
+ multiple_imports=dotted_as_names<
+ any* dotted_as_name< (%s) 'as' any > any* >) >
+ """ % (mod_list, mod_list)
 
 # Find usages of module members in code e.g. thread.foo(bar)
 yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names
@@ -100,8 +103,8 @@
 match = super(FixImports, self).match
 results = match(node)
 if results:
- # Module usage could be in the trailier of an attribute lookup, so
- # we might have nested matches when "bare_with_attr" is present.
+ # Module usage could be in the trailer of an attribute lookup, so we
+ # might have nested matches when "bare_with_attr" is present.
 if "bare_with_attr" not in results and \
 any([match(obj) for obj in attr_chain(node, "parent")]):
 return False
@@ -116,11 +119,21 @@
 import_mod = results.get("module_name")
 if import_mod:
 new_name = self.mapping[(import_mod or mod_name).value]
+ import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
 if "name_import" in results:
 # If it's not a "from x import x, y" or "import x as y" import,
 # marked its usage to be replaced.
 self.replace[import_mod.value] = new_name
- import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
+ if "multiple_imports" in results:
+ # This is a nasty hack to fix multiple imports on a
+ # line (e.g., "import StringIO, urlparse"). The problem is that I
+ # can't figure out an easy way to make a pattern recognize the
+ # keys of MAPPING randomly sprinkled in an import statement.
+ while True:
+ results = self.match(node)
+ if not results:
+ break
+ self.transform(node, results)
 else:
 # Replace usage of the module.
 bare_name = results["bare_with_attr"][0]
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports2.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports2.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_imports2.py	Tue Dec 16 05:13:50 2008
@@ -11,6 +11,6 @@
 
 class FixImports2(fix_imports.FixImports):
 
- order = "post"
+ run_order = 6
 
 mapping = MAPPING
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_intern.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_intern.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_intern.py	Tue Dec 16 05:13:50 2008
@@ -8,7 +8,7 @@
 # Local imports
 from .. import pytree
 from .. import fixer_base
-from ..fixer_util import Name, Attr
+from ..fixer_util import Name, Attr, touch_import
 
 
 class FixIntern(fixer_base.BaseFix):
@@ -40,4 +40,5 @@
 newarglist,
 results["rpar"].clone()])] + after)
 new.set_prefix(node.get_prefix())
+ touch_import(None, 'sys', node)
 return new
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_long.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_long.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_long.py	Tue Dec 16 05:13:50 2008
@@ -2,8 +2,6 @@
 # Licensed to PSF under a Contributor Agreement.
 
 """Fixer that turns 'long' into 'int' everywhere.
-
-This also strips the trailing 'L' or 'l' from long loterals.
 """
 
 # Local imports
@@ -14,22 +12,13 @@
 
 class FixLong(fixer_base.BaseFix):
 
- PATTERN = """
- (long_type = 'long' | number = NUMBER)
- """
+ PATTERN = "'long'"
 
 static_long = Name("long")
 static_int = Name("int")
 
 def transform(self, node, results):
- long_type = results.get("long_type")
- number = results.get("number")
- new = None
- if long_type:
- assert node == self.static_long, node
- new = self.static_int.clone()
- if number and node.value[-1] in ("l", "L"):
- new = Number(node.value[:-1])
- if new is not None:
- new.set_prefix(node.get_prefix())
- return new
+ assert node == self.static_long, node
+ new = self.static_int.clone()
+ new.set_prefix(node.get_prefix())
+ return new
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_repr.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_repr.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_repr.py	Tue Dec 16 05:13:50 2008
@@ -5,7 +5,7 @@
 
 # Local imports
 from .. import fixer_base
-from ..fixer_util import Call, Name
+from ..fixer_util import Call, Name, parenthesize
 
 
 class FixRepr(fixer_base.BaseFix):
@@ -18,5 +18,5 @@
 expr = results["expr"].clone()
 
 if expr.type == self.syms.testlist1:
- expr = self.parenthesize(expr)
+ expr = parenthesize(expr)
 return Call(Name("repr"), [expr], prefix=node.get_prefix())
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_urllib.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_urllib.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_urllib.py	Tue Dec 16 05:13:50 2008
@@ -29,7 +29,7 @@
 'AbstractBasicAuthHandler',
 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler',
 'AbstractDigestAuthHandler',
- 'HTTPDigestAuthHander', 'ProxyDigestAuthHandler',
+ 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler',
 'HTTPHandler', 'HTTPSHandler', 'FileHandler',
 'FTPHandler', 'CacheFTPHandler',
 'UnknownHandler']),
Modified: python/branches/release30-maint/Lib/lib2to3/fixes/fix_xrange.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/fixes/fix_xrange.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/fixes/fix_xrange.py	Tue Dec 16 05:13:50 2008
@@ -12,7 +12,9 @@
 class FixXrange(fixer_base.BaseFix):
 
 PATTERN = """
- power< (name='range'|name='xrange') trailer< '(' [any] ')' > any* >
+ power<
+ (name='range'|name='xrange') trailer< '(' args=any ')' >
+ rest=any* >
 """
 
 def transform(self, node, results):
@@ -30,11 +32,14 @@
 
 def transform_range(self, node, results):
 if not self.in_special_context(node):
- arg = node.clone()
- arg.set_prefix("")
- call = Call(Name("list"), [arg])
- call.set_prefix(node.get_prefix())
- return call
+ range_call = Call(Name("range"), [results["args"].clone()])
+ # Encase the range call in list().
+ list_call = Call(Name("list"), [range_call],
+ prefix=node.get_prefix())
+ # Put things that were after the range() call after the list call.
+ for n in results["rest"]:
+ list_call.append_child(n)
+ return list_call
 return node
 
 P1 = "power< func=NAME trailer< '(' node=any ')' > any* >"
Modified: python/branches/release30-maint/Lib/lib2to3/main.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/main.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/main.py	Tue Dec 16 05:13:50 2008
@@ -5,6 +5,7 @@
 import sys
 import os
 import logging
+import shutil
 import optparse
 
 from . import refactor
@@ -39,6 +40,7 @@
 # Actually write the new file
 super(StdoutRefactoringTool, self).write_file(new_text,
 filename, old_text)
+ shutil.copymode(filename, backup)
 
 def print_output(self, lines):
 for line in lines:
@@ -56,7 +58,7 @@
 Returns a suggested exit status (0, 1, 2).
 """
 # Set up option parser
- parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...")
+ parser = optparse.OptionParser(usage="2to3 [options] file|dir ...")
 parser.add_option("-d", "--doctests_only", action="store_true",
 help="Fix up doctests only")
 parser.add_option("-f", "--fix", action="append", default=[],
Modified: python/branches/release30-maint/Lib/lib2to3/pygram.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/pygram.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/pygram.py	Tue Dec 16 05:13:50 2008
@@ -29,10 +29,3 @@
 
 python_grammar = driver.load_grammar(_GRAMMAR_FILE)
 python_symbols = Symbols(python_grammar)
-
-
-def parenthesize(node):
- return pytree.Node(python_symbols.atom,
- (pytree.Leaf(token.LPAR, "("),
- node,
- pytree.Leaf(token.RPAR, ")")))
Modified: python/branches/release30-maint/Lib/lib2to3/pytree.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/pytree.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/pytree.py	Tue Dec 16 05:13:50 2008
@@ -279,18 +279,21 @@
 child.parent = self
 self.children[i].parent = None
 self.children[i] = child
+ self.changed()
 
 def insert_child(self, i, child):
 """Equivalent to 'node.children.insert(i, child)'. This method also
 sets the child's parent attribute appropriately."""
 child.parent = self
 self.children.insert(i, child)
+ self.changed()
 
 def append_child(self, child):
 """Equivalent to 'node.children.append(child)'. This method also
 sets the child's parent attribute appropriately."""
 child.parent = self
 self.children.append(child)
+ self.changed()
 
 
 class Leaf(Base):
Modified: python/branches/release30-maint/Lib/lib2to3/refactor.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/refactor.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/refactor.py	Tue Dec 16 05:13:50 2008
@@ -123,8 +123,8 @@
 logger=self.logger)
 self.pre_order, self.post_order = self.get_fixers()
 
- self.pre_order_mapping = get_headnode_dict(self.pre_order)
- self.post_order_mapping = get_headnode_dict(self.post_order)
+ self.pre_order_heads = get_headnode_dict(self.pre_order)
+ self.post_order_heads = get_headnode_dict(self.post_order)
 
 self.files = [] # List of files that were or should be modified
 
@@ -294,8 +294,8 @@
 for fixer in all_fixers:
 fixer.start_tree(tree, name)
 
- self.traverse_by(self.pre_order_mapping, tree.pre_order())
- self.traverse_by(self.post_order_mapping, tree.post_order())
+ self.traverse_by(self.pre_order_heads, tree.pre_order())
+ self.traverse_by(self.post_order_heads, tree.post_order())
 
 for fixer in all_fixers:
 fixer.finish_tree(tree, name)
Deleted: python/branches/release30-maint/Lib/lib2to3/tests/benchmark.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/tests/benchmark.py	Tue Dec 16 05:13:50 2008
+++ (empty file)
@@ -1,58 +0,0 @@
-#!/usr/bin/env python2.5
-"""
-This is a benchmarking script to test the speed of 2to3's pattern matching
-system. It's equivalent to "refactor.py -f all" for every Python module
-in sys.modules, but without engaging the actual transformations.
-"""
-
-__author__ = "Collin Winter <collinw at gmail.com>"
-
-# Python imports
-import os.path
-import sys
-from time import time
-
-# Test imports
-from .support import adjust_path
-adjust_path()
-
-# Local imports
-from .. import refactor
-
-### Mock code for refactor.py and the fixers
-###############################################################################
-class Options:
- def __init__(self, **kwargs):
- for k, v in list(kwargs.items()):
- setattr(self, k, v)
-
- self.verbose = False
-
-def dummy_transform(*args, **kwargs):
- pass
-
-### Collect list of modules to match against
-###############################################################################
-files = []
-for mod in list(sys.modules.values()):
- if mod is None or not hasattr(mod, '__file__'):
- continue
- f = mod.__file__
- if f.endswith('.pyc'):
- f = f[:-1]
- if f.endswith('.py'):
- files.append(f)
-
-### Set up refactor and run the benchmark
-###############################################################################
-options = Options(fix=["all"], print_function=False, doctests_only=False)
-refactor = refactor.RefactoringTool(options)
-for fixer in refactor.fixers:
- # We don't want them to actually fix the tree, just match against it.
- fixer.transform = dummy_transform
-
-t = time()
-for f in files:
- print("Matching", f)
- refactor.refactor_file(f)
-print("%d seconds to match %d files" % (time() - t, len(sys.modules)))
Modified: python/branches/release30-maint/Lib/lib2to3/tests/test_fixers.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/tests/test_fixers.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/tests/test_fixers.py	Tue Dec 16 05:13:50 2008
@@ -293,30 +293,30 @@
 
 def test_prefix_preservation(self):
 b = """x = intern( a )"""
- a = """x = sys.intern( a )"""
+ a = """import sys\nx = sys.intern( a )"""
 self.check(b, a)
 
 b = """y = intern("b" # test
 )"""
- a = """y = sys.intern("b" # test
+ a = """import sys\ny = sys.intern("b" # test
 )"""
 self.check(b, a)
 
 b = """z = intern(a+b+c.d, )"""
- a = """z = sys.intern(a+b+c.d, )"""
+ a = """import sys\nz = sys.intern(a+b+c.d, )"""
 self.check(b, a)
 
 def test(self):
 b = """x = intern(a)"""
- a = """x = sys.intern(a)"""
+ a = """import sys\nx = sys.intern(a)"""
 self.check(b, a)
 
 b = """z = intern(a+b+c.d,)"""
- a = """z = sys.intern(a+b+c.d,)"""
+ a = """import sys\nz = sys.intern(a+b+c.d,)"""
 self.check(b, a)
 
 b = """intern("y%s" % 5).replace("y", "")"""
- a = """sys.intern("y%s" % 5).replace("y", "")"""
+ a = """import sys\nsys.intern("y%s" % 5).replace("y", "")"""
 self.check(b, a)
 
 # These should not be refactored
@@ -337,6 +337,35 @@
 s = """intern()"""
 self.unchanged(s)
 
+class Test_reduce(FixerTestCase):
+ fixer = "reduce"
+
+ def test_simple_call(self):
+ b = "reduce(a, b, c)"
+ a = "from functools import reduce\nreduce(a, b, c)"
+ self.check(b, a)
+
+ def test_call_with_lambda(self):
+ b = "reduce(lambda x, y: x + y, seq)"
+ a = "from functools import reduce\nreduce(lambda x, y: x + y, seq)"
+ self.check(b, a)
+
+ def test_unchanged(self):
+ s = "reduce(a)"
+ self.unchanged(s)
+
+ s = "reduce(a, b=42)"
+ self.unchanged(s)
+
+ s = "reduce(a, b, c, d)"
+ self.unchanged(s)
+
+ s = "reduce(**c)"
+ self.unchanged(s)
+
+ s = "reduce()"
+ self.unchanged(s)
+
 class Test_print(FixerTestCase):
 fixer = "print"
 
@@ -1044,33 +1073,39 @@
 a = """z = type(x) in (int, int)"""
 self.check(b, a)
 
- def test_4(self):
- b = """a = 12L"""
- a = """a = 12"""
+ def test_prefix_preservation(self):
+ b = """x = long( x )"""
+ a = """x = int( x )"""
 self.check(b, a)
 
- def test_5(self):
- b = """b = 0x12l"""
- a = """b = 0x12"""
+class Test_isinstance(FixerTestCase):
+ fixer = "isinstance"
+
+ def test_remove_multiple_items(self):
+ b = """isinstance(x, (int, int, int))"""
+ a = """isinstance(x, int)"""
 self.check(b, a)
 
- def test_unchanged_1(self):
- s = """a = 12"""
- self.unchanged(s)
+ b = """isinstance(x, (int, float, int, int, float))"""
+ a = """isinstance(x, (int, float))"""
+ self.check(b, a)
 
- def test_unchanged_2(self):
- s = """b = 0x12"""
- self.unchanged(s)
+ b = """isinstance(x, (int, float, int, int, float, str))"""
+ a = """isinstance(x, (int, float, str))"""
+ self.check(b, a)
 
- def test_unchanged_3(self):
- s = """c = 3.14"""
- self.unchanged(s)
+ b = """isinstance(foo() + bar(), (x(), y(), x(), int, int))"""
+ a = """isinstance(foo() + bar(), (x(), y(), x(), int))"""
+ self.check(b, a)
 
 def test_prefix_preservation(self):
- b = """x = long( x )"""
- a = """x = int( x )"""
+ b = """if isinstance( foo(), ( bar, bar, baz )) : pass"""
+ a = """if isinstance( foo(), ( bar, baz )) : pass"""
 self.check(b, a)
 
+ def test_unchanged(self):
+ self.unchanged("isinstance(x, (str, int))")
+
 class Test_dict(FixerTestCase):
 fixer = "dict"
 
@@ -1287,6 +1322,14 @@
 a = """x = list(range(10, 3, 9)) + [4]"""
 self.check(b, a)
 
+ b = """x = range(10)[::-1]"""
+ a = """x = list(range(10))[::-1]"""
+ self.check(b, a)
+
+ b = """x = range(10) [3]"""
+ a = """x = list(range(10)) [3]"""
+ self.check(b, a)
+
 def test_xrange_in_for(self):
 b = """for i in xrange(10):\n j=i"""
 a = """for i in range(10):\n j=i"""
@@ -1422,9 +1465,8 @@
 s = "foo(xreadlines)"
 self.unchanged(s)
 
-class Test_imports(FixerTestCase):
- fixer = "imports"
- from ..fixes.fix_imports import MAPPING as modules
+
+class ImportsFixerTests:
 
 def test_import_module(self):
 for old, new in self.modules.items():
@@ -1522,18 +1564,36 @@
 self.check(b, a)
 
 
+class Test_imports(FixerTestCase, ImportsFixerTests):
+ fixer = "imports"
+ from ..fixes.fix_imports import MAPPING as modules
+
+ def test_multiple_imports(self):
+ b = """import urlparse, cStringIO"""
+ a = """import urllib.parse, io"""
+ self.check(b, a)
+
+ def test_multiple_imports_as(self):
+ b = """
+ import copy_reg as bar, HTMLParser as foo, urlparse
+ s = urlparse.spam(bar.foo())
+ """
+ a = """
+ import copyreg as bar, html.parser as foo, urllib.parse
+ s = urllib.parse.spam(bar.foo())
+ """
+ self.check(b, a)
+
 
-class Test_imports2(Test_imports):
+class Test_imports2(FixerTestCase, ImportsFixerTests):
 fixer = "imports2"
 from ..fixes.fix_imports2 import MAPPING as modules
 
 
-class Test_imports_fixer_order(Test_imports):
-
- fixer = None
+class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests):
 
 def setUp(self):
- Test_imports.setUp(self, ['imports', 'imports2'])
+ super(Test_imports_fixer_order, self).setUp(['imports', 'imports2'])
 from ..fixes.fix_imports2 import MAPPING as mapping2
 self.modules = mapping2.copy()
 from ..fixes.fix_imports import MAPPING as mapping1
Modified: python/branches/release30-maint/Lib/lib2to3/tests/test_util.py
==============================================================================
--- python/branches/release30-maint/Lib/lib2to3/tests/test_util.py	(original)
+++ python/branches/release30-maint/Lib/lib2to3/tests/test_util.py	Tue Dec 16 05:13:50 2008
@@ -526,6 +526,33 @@
 b = 7"""
 self.failIf(self.find_binding("a", s))
 
+class Test_touch_import(support.TestCase):
+
+ def test_after_docstring(self):
+ node = parse('"""foo"""\nbar()')
+ fixer_util.touch_import(None, "foo", node)
+ self.assertEqual(str(node), '"""foo"""\nimport foo\nbar()\n\n')
+
+ def test_after_imports(self):
+ node = parse('"""foo"""\nimport bar\nbar()')
+ fixer_util.touch_import(None, "foo", node)
+ self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n')
+
+ def test_beginning(self):
+ node = parse('bar()')
+ fixer_util.touch_import(None, "foo", node)
+ self.assertEqual(str(node), 'import foo\nbar()\n\n')
+
+ def test_from_import(self):
+ node = parse('bar()')
+ fixer_util.touch_import("cgi", "escape", node)
+ self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n')
+
+ def test_name_import(self):
+ node = parse('bar()')
+ fixer_util.touch_import(None, "cgi", node)
+ self.assertEqual(str(node), 'import cgi\nbar()\n\n')
+
 
 if __name__ == "__main__":
 import __main__
