Merge branch 'develop' into multiple-hashers

Conflicts:
	pip/backwardcompat.py
	pip/download.py

.travis.yml

@@ -11,6 +11,7 @@ before_install:
 install: pip install nose virtualenv scripttest mock
 script: nosetests
 notifications:
+  irc: "irc.freenode.org#pip"
 branches:
   only:
     - develop

AUTHORS.txt

@@ -28,6 +28,7 @@ Luke Macken
 Masklinn
 Marc Abramowitz
 Marcus Smith
+Markus Hametner
 Matt Maker
 Nick Stenning
 Nowell Strite
@@ -51,3 +52,4 @@ Vinay Sajip
 Vitaly Babiy
 W Trevor King
 Wil Tan
+Hsiaoming Yang

docs/conf.py

@@ -47,7 +47,7 @@ copyright = '2008-2011, The pip developers'
 # built documents.
 #
 # The short X.Y version.
-release = "1.1.post1"
+release = "1.1.post2"
 version = '.'.join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/news.txt

@@ -13,6 +13,15 @@ Beta and final releases planned for the second half of 2012.
 develop (unreleased)
 -------------------
 
+* Write failure log to temp file if default location is not writable. Thanks
+  andreigc.
+
+* Pull in submodules for git editable checkouts. Fixes #289 and #421. Thanks
+  Hsiaoming Yang and Markus Hametner.
+
+* Use a temporary directory as the default build location outside of a
+  virtualenv. Fixes issues #339 and #381. Thanks TC01.
+
 * Added support for specifying extras with local editables. Thanks Nick
   Stenning.
 

docs/requirements.txt

@@ -100,6 +100,8 @@ installed.  For example::
 
 tells pip to install the 3.0 version of MyPackage.
 
+A line beginning with ``#`` is treated as a comment and ignored.
+
 You can also request `extras`_ in the requirements file::
 
     MyPackage==3.0 [PDF]

pip/__init__.py

@@ -2,17 +2,16 @@
 import os
 import optparse
 
-import subprocess
 import sys
 import re
 import difflib
 
-from pip.backwardcompat import walk_packages
 from pip.basecommand import command_dict, load_command, load_all_commands, command_names
 from pip.baseparser import parser
 from pip.exceptions import InstallationError
 from pip.log import logger
 from pip.util import get_installed_distributions
+from pip.vcs import git, mercurial, subversion, bazaar
 
 
 def autocomplete():
@@ -80,19 +79,10 @@ def autocomplete():
     sys.exit(1)
 
 
-def version_control():
-    # Import all the version control support modules:
-    from pip import vcs
-    for importer, modname, ispkg in \
-            walk_packages(path=vcs.__path__, prefix=vcs.__name__+'.'):
-        __import__(modname)
-
-
 def main(initial_args=None):
     if initial_args is None:
         initial_args = sys.argv[1:]
     autocomplete()
-    version_control()
     options, args = parser.parse_args(initial_args)
     if options.help and not args:
         args = ['help']

pip/backwardcompat.py

 """Stuff that differs in different Python versions"""
 
 import sys
-import os
-import shutil
+import site
 
-__all__ = ['any', 'WindowsError', 'copytree']
+__all__ = ['WindowsError']
 
 try:
     WindowsError = WindowsError
@@ -13,21 +12,6 @@ except NameError:
         """this exception should never be raised"""
     WindowsError = NeverUsedException
 
-try:
-    from pkgutil import walk_packages
-except ImportError:
-    # let's fall back as long as we can
-    from pip._pkgutil import walk_packages
-
-try:
-    any = any
-except NameError:
-
-    def any(seq):
-        for item in seq:
-            if item:
-                return True
-        return False
 
 console_encoding = sys.__stdout__.encoding
 
@@ -99,25 +83,11 @@ else:
     raw_input = raw_input
     BytesIO = StringIO
 
-try:
-    from email.parser import FeedParser
-except ImportError:
-    # python lesser than 2.5
-    from email.FeedParser import FeedParser
 
 from distutils.sysconfig import get_python_lib, get_python_version
 
-
-def copytree(src, dst):
-    if sys.version_info < (2, 5):
-        before_last_dir = os.path.dirname(dst)
-        if not os.path.exists(before_last_dir):
-            os.makedirs(before_last_dir)
-        shutil.copytree(src, dst)
-        shutil.copymode(src, dst)
-    else:
-        shutil.copytree(src, dst)
-
+#site.USER_SITE was created in py2.6
+user_site = getattr(site,'USER_SITE',None)
 
 def product(*args, **kwds):
     # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy

pip/basecommand.py

 """Base Command class, and related routines"""
 
 import os
+from pkgutil import walk_packages
 import socket
 import sys
 import traceback
@@ -12,7 +13,7 @@ from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpForma
 from pip.download import urlopen
 from pip.exceptions import (BadCommand, InstallationError, UninstallationError,
                             CommandError)
-from pip.backwardcompat import StringIO, walk_packages
+from pip.backwardcompat import StringIO
 from pip.status_codes import SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND
 
 
@@ -137,8 +138,13 @@ class Command(object):
         if store_log:
             log_fn = options.log_file
             text = '\n'.join(complete_log)
-            logger.fatal('Storing complete log in %s' % log_fn)
-            log_fp = open_logfile(log_fn, 'w')
+            try:
+               log_fp = open_logfile(log_fn, 'w')
+            except IOError:
+               temp = tempfile.NamedTemporaryFile(delete=False)
+               log_fn = temp.name
+               log_fp = open_logfile(log_fn, 'w')
+            logger.fatal('Storing complete log in %s' % log_fn)			
             log_fp.write(text)
             log_fp.close()
         return exit

pip/commands/install.py

@@ -5,7 +5,7 @@ import shutil
 from pip.req import InstallRequirement, RequirementSet
 from pip.req import parse_requirements
 from pip.log import logger
-from pip.locations import build_prefix, src_prefix
+from pip.locations import build_prefix, src_prefix, virtualenv_no_global
 from pip.basecommand import Command
 from pip.index import PackageFinder
 from pip.exceptions import InstallationError, CommandError
@@ -190,6 +190,8 @@ class InstallCommand(Command):
         options.src_dir = os.path.abspath(options.src_dir)
         install_options = options.install_options or []
         if options.use_user_site:
+            if virtualenv_no_global():
+                raise InstallationError("Can not perform a '--user' install. User site-packages are not visible in this virtualenv.")
             install_options.append('--user')
         if options.target_dir:
             options.ignore_installed = True
@@ -215,7 +217,8 @@ class InstallCommand(Command):
             as_egg=options.as_egg,
             ignore_installed=options.ignore_installed,
             ignore_dependencies=options.ignore_dependencies,
-            force_reinstall=options.force_reinstall)
+            force_reinstall=options.force_reinstall,
+            use_user_site=options.use_user_site)
         for name in args:
             requirement_set.add_requirement(
                 InstallRequirement.from_line(name, None))

pip/download.py

@@ -7,11 +7,11 @@ import re
 import shutil
 import sys
 import tempfile
-from pip.backwardcompat import (copytree, xmlrpclib, urllib, urllib2,
-                                urlparse, string_types, HTTPError)
+from pip.backwardcompat import (xmlrpclib, urllib, urllib2,
+                                urlparse, string_types)
 from pip.exceptions import InstallationError
 from pip.util import (splitext, rmtree, format_size, display_path,
-                      backup_dir, ask, ask_path_exists, unpack_file,
+                      backup_dir, ask_path_exists, unpack_file,
                       create_download_cache_folder, cache_download)
 from pip.vcs import vcs
 from pip.log import logger
@@ -302,7 +302,7 @@ def unpack_file_url(link, location):
         # delete the location since shutil will create it again :(
         if os.path.isdir(location):
             rmtree(location)
-        copytree(source, location)
+        shutil.copytree(source, location)
     else:
         unpack_file(source, location, content_type, link)
 

pip/locations.py

 """Locations where we look for configs, install stuff, etc"""
 
 import sys
+import site
 import os
+import tempfile
 from pip.backwardcompat import get_python_lib
 
 
@@ -12,15 +14,28 @@ def running_under_virtualenv():
     """
     return hasattr(sys, 'real_prefix')
 
+def virtualenv_no_global():
+    """
+    Return True if in a venv and no system site packages.
+    """
+    #this mirrors the logic in virtualenv.py for locating the no-global-site-packages.txt file
+    site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
+    no_global_file = os.path.join(site_mod_dir,'no-global-site-packages.txt')
+    if running_under_virtualenv() and os.path.isfile(no_global_file):
+        return True
+
 
 if running_under_virtualenv():
     ## FIXME: is build/ a good name?
     build_prefix = os.path.join(sys.prefix, 'build')
     src_prefix = os.path.join(sys.prefix, 'src')
 else:
+    # Use tempfile to create a temporary folder for build
+    # Note: we are NOT using mkdtemp so we can have a consistent build dir
+    build_prefix = os.path.join(tempfile.gettempdir(), 'pip-build')
+    
+    ## FIXME: keep src in cwd for now (it is not a temporary folder)
     try:
-        ## FIXME: this isn't a very good default
-        build_prefix = os.path.join(os.getcwd(), 'build')
         src_prefix = os.path.join(os.getcwd(), 'src')
     except OSError:
         # In case the current working directory has been renamed or deleted

pip/req.py

-import sys
+from email.parser import FeedParser
 import os
-import shutil
-import re
-import zipfile
 import pkg_resources
+import re
+import sys
+import shutil
 import tempfile
+import zipfile
+
 from pip.locations import bin_py, running_under_virtualenv
 from pip.exceptions import (InstallationError, UninstallationError,
                             BestVersionAlreadyInstalled,
@@ -13,14 +15,13 @@ from pip.vcs import vcs
 from pip.log import logger
 from pip.util import display_path, rmtree
 from pip.util import ask, ask_path_exists, backup_dir
-from pip.util import is_installable_dir, is_local, dist_is_local
+from pip.util import is_installable_dir, is_local, dist_is_local, dist_in_usersite
 from pip.util import renames, normalize_path, egg_link_path
 from pip.util import make_path_relative
 from pip.util import call_subprocess
-from pip.backwardcompat import (any, copytree, urlparse, urllib,
+from pip.backwardcompat import (urlparse, urllib,
                                 ConfigParser, string_types, HTTPError,
-                                FeedParser, get_python_version,
-                                b)
+                                get_python_version, b)
 from pip.index import Link
 from pip.locations import build_prefix
 from pip.download import (get_file_content, is_url, url_to_path,
@@ -61,6 +62,7 @@ class InstallRequirement(object):
         self.install_succeeded = None
         # UninstallPathSet of uninstalled distribution (for possible rollback)
         self.uninstalled = None
+        self.use_user_site = False
 
     @classmethod
     def from_editable(cls, editable_req, comes_from=None, default_vcs=None):
@@ -563,7 +565,7 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
         if self.editable:
             self.install_editable(install_options, global_options)
             return
-        
+
         temp_location = tempfile.mkdtemp('-record', 'pip-')
         record_filename = os.path.join(temp_location, 'install-record.txt')
         try:
@@ -679,7 +681,12 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
         except pkg_resources.DistributionNotFound:
             return False
         except pkg_resources.VersionConflict:
-            self.conflicts_with = pkg_resources.get_distribution(self.req.project_name)
+            existing_dist = pkg_resources.get_distribution(self.req.project_name)
+            if self.use_user_site:
+                if dist_in_usersite(existing_dist):
+                    self.conflicts_with = existing_dist
+            else:
+                self.conflicts_with = existing_dist
         return True
 
     @property
@@ -797,7 +804,7 @@ class RequirementSet(object):
 
     def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
                  upgrade=False, ignore_installed=False, as_egg=False,
-                 ignore_dependencies=False, force_reinstall=False):
+                 ignore_dependencies=False, force_reinstall=False, use_user_site=False):
         self.build_dir = build_dir
         self.src_dir = src_dir
         self.download_dir = download_dir
@@ -814,6 +821,7 @@ class RequirementSet(object):
         self.successfully_installed = []
         self.reqs_to_cleanup = []
         self.as_egg = as_egg
+        self.use_user_site = use_user_site
 
     def __str__(self):
         reqs = [req for req in self.requirements.values()
@@ -824,6 +832,7 @@ class RequirementSet(object):
     def add_requirement(self, install_req):
         name = install_req.name
         install_req.as_egg = self.as_egg
+        install_req.use_user_site = self.use_user_site
         if not name:
             self.unnamed_requirements.append(install_req)
         else:
@@ -1113,7 +1122,7 @@ class RequirementSet(object):
         target_dir = req_to_install.editable and self.src_dir or self.build_dir
         logger.info("Copying %s to %s" % (req_to_install.name, target_dir))
         dest = os.path.join(target_dir, req_to_install.name)
-        copytree(req_to_install.source_dir, dest)
+        shutil.copytree(req_to_install.source_dir, dest)
         call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest,
                         command_desc='python setup.py clean')
 
@@ -1336,7 +1345,9 @@ def parse_editable(editable_req, default_vcs=None):
     else:
         url_no_extras = url
 
-    if os.path.isdir(url_no_extras) and os.path.exists(os.path.join(url_no_extras, 'setup.py')):
+    if os.path.isdir(url_no_extras):
+        if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
+            raise InstallationError("Directory %r is not installable. File 'setup.py' not found.", url_no_extras)
         # Treating it as code that has already been checked out
         url_no_extras = path_to_url(url_no_extras)
 
@@ -1357,8 +1368,10 @@ def parse_editable(editable_req, default_vcs=None):
                 '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req)
     vc_type = url.split('+', 1)[0].lower()
     if not vcs.get_backend(vc_type):
-        raise InstallationError(
-            'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req)
+        error_message = 'For --editable=%s only ' % editable_req + \
+            ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
+            ' is currently supported'
+        raise InstallationError(error_message)
     match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req)
     if (not match or not match.group(1)) and vcs.get_backend(vc_type):
         parts = [p for p in editable_req.split('#', 1)[0].split('/') if p]

pip/util.py

@@ -9,7 +9,7 @@ import zipfile
 import tarfile
 import subprocess
 from pip.exceptions import InstallationError, BadCommand
-from pip.backwardcompat import WindowsError, string_types, raw_input, console_to_str
+from pip.backwardcompat import WindowsError, string_types, raw_input, console_to_str, user_site
 from pip.locations import site_packages, running_under_virtualenv
 from pip.log import logger
 
@@ -294,6 +294,16 @@ def dist_is_local(dist):
     return is_local(dist_location(dist))
 
 
+def dist_in_usersite(dist):
+    """
+    Return True if given Distribution is installed in user site.
+    """
+    if user_site:
+        return normalize_path(dist_location(dist)).startswith(normalize_path(user_site))
+    else:
+        return False
+
+
 def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python')):
     """
     Return a list of installed Distribution objects.

pip/vcs/__init__.py

@@ -19,7 +19,9 @@ class VcsSupport(object):
     def __init__(self):
         # Register more schemes with urlparse for various version control systems
         urlparse.uses_netloc.extend(self.schemes)
-        urlparse.uses_fragment.extend(self.schemes)
+        # Python 3.3 doesn't have uses_fragment
+        if getattr(urlparse, 'uses_fragment', None):
+            urlparse.uses_fragment.extend(self.schemes)
         super(VcsSupport, self).__init__()
 
     def __iter__(self):

pip/vcs/git.py

 import tempfile
 import re
+import os.path
 from pip.util import call_subprocess
 from pip.util import display_path, rmtree
 from pip.vcs import vcs, VersionControl
@@ -28,8 +29,8 @@ class Git(VersionControl):
                 initial_slashes = path[:-len(path.lstrip('/'))]
                 newpath = initial_slashes + url2pathname(path).replace('\\', '/').lstrip('/')
                 url = urlunsplit((scheme, netloc, newpath, query, fragment))
-                after_plus = scheme.find('+')+1
-                url = scheme[:after_plus]+ urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment))
+                after_plus = scheme.find('+') + 1
+                url = scheme[:after_plus] + urlunsplit((scheme[after_plus:], netloc, newpath, query, fragment))
 
         super(Git, self).__init__(url, *args, **kwargs)
 
@@ -86,6 +87,8 @@ class Git(VersionControl):
         call_subprocess(
             [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
 
+        self.update_submodules(dest)
+
     def update(self, dest, rev_options):
         # First fetch changes from the default remote
         call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest)
@@ -93,6 +96,8 @@ class Git(VersionControl):
         if rev_options:
             rev_options = self.check_rev_options(rev_options[0], dest, rev_options)
         call_subprocess([self.cmd, 'reset', '--hard', '-q'] + rev_options, cwd=dest)
+        #: update submodules
+        self.update_submodules(dest)
 
     def obtain(self, dest):
         url, rev = self.get_url_rev()
@@ -105,6 +110,8 @@ class Git(VersionControl):
         if self.check_destination(dest, url, rev_options, rev_display):
             logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest)))
             call_subprocess([self.cmd, 'clone', '-q', url, dest])
+            #: repo may contain submodules
+            self.update_submodules(dest)
             if rev:
                 rev_options = self.check_rev_options(rev, dest, rev_options)
                 # Only do a checkout if rev_options differs from HEAD
@@ -161,8 +168,10 @@ class Git(VersionControl):
         elif (current_rev in branch_revs and
               branch_revs[current_rev] != 'origin/master'):
             # It's the head of a branch
-            full_egg_name = '%s-%s' % (egg_project_name,
-                                       branch_revs[current_rev].replace('origin/', ''))
+            full_egg_name = '%s-%s' % (
+                egg_project_name,
+                branch_revs[current_rev].replace('origin/', '')
+            )
         else:
             full_egg_name = '%s-dev' % egg_project_name
 
@@ -202,5 +211,11 @@ class Git(VersionControl):
         return call_subprocess([self.cmd, 'rev-parse', name],
                                show_stdout=False, cwd=location)
 
+    def update_submodules(self, location):
+        if not os.path.exists(os.path.join(location, '.gitmodules')):
+            return
+        call_subprocess([self.cmd, 'submodule', 'init', '-q'], cwd=location)
+        call_subprocess([self.cmd, 'submodule', 'update', '--recursive', '-q'],
+                        cwd=location)
 
 vcs.register(Git)

setup.py

@@ -3,7 +3,7 @@ import os
 from setuptools import setup
 
 # If you change this version, change it also in docs/conf.py
-version = "1.1.post1"
+version = "1.1.post2"
 
 doc_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "docs")
 index_filename = os.path.join(doc_dir, "index.txt")

tests/git_submodule_helpers.py

+import textwrap
+from tests.test_pip import (mkdir, write_file,)
+
+def _create_test_package_submodule(env):
+    mkdir('version_pkg_submodule')
+    submodule_path = env.scratch_path/'version_pkg_submodule'
+    env.run('touch', 'testfile', cwd=submodule_path)
+    env.run('git', 'init', cwd=submodule_path)
+    env.run('git', 'add', '.', cwd=submodule_path)
+    env.run('git', 'commit', '-q',
+            '--author', 'Pip <python-virtualenv@googlegroups.com>',
+            '-am', 'initial version / submodule', cwd=submodule_path)
+    return submodule_path
+
+def _change_test_package_submodule(env, submodule_path):
+    write_file(submodule_path/'testfile', 'this is a changed file')
+    write_file(submodule_path/'testfile2', 'this is an added file')
+    env.run('git', 'add', '.', cwd=submodule_path)
+    env.run('git', 'commit', '-q',
+            '--author', 'Pip <python-virtualenv@googlegroups.com>',
+            '-am', 'submodule change', cwd=submodule_path)
+
+def _pull_in_submodule_changes_to_module(env, module_path):
+    env.run(cwd=module_path/'testpkg/static/', *('git pull -q origin master'.split(' ')))
+    env.run('git', 'commit', '-q',
+            '--author', 'Pip <python-virtualenv@googlegroups.com>',
+            '-am', 'submodule change', cwd=module_path)
+
+def _create_test_package_with_submodule(env):
+    mkdir('version_pkg')
+    version_pkg_path = env.scratch_path/'version_pkg'
+    mkdir(version_pkg_path/'testpkg')
+    pkg_path = version_pkg_path/'testpkg'
+
+    write_file('__init__.py', '# hello there', pkg_path)
+    write_file('version_pkg.py', textwrap.dedent('''\
+                                def main():
+                                    print('0.1')
+                                '''), pkg_path)
+    write_file('setup.py', textwrap.dedent('''\
+                        from setuptools import setup, find_packages
+                        setup(name='version_pkg',
+                              version='0.1',
+                              packages=find_packages(),
+                             )
+                        '''), version_pkg_path)
+    env.run('git', 'init', cwd=version_pkg_path)
+    env.run('git', 'add', '.', cwd=version_pkg_path)
+    env.run('git', 'commit', '-q',
+            '--author', 'Pip <python-virtualenv@googlegroups.com>',
+            '-am', 'initial version', cwd=version_pkg_path)
+
+
+    submodule_path = _create_test_package_submodule(env)
+
+    env.run('git', 'submodule', 'add', submodule_path, 'testpkg/static', cwd=version_pkg_path)
+    env.run('git', 'commit', '-q',
+            '--author', 'Pip <python-virtualenv@googlegroups.com>',
+            '-am', 'initial version w submodule', cwd=version_pkg_path)
+
+
+    return version_pkg_path, submodule_path

tests/test_basic.py

@@ -41,7 +41,7 @@ def test_correct_pip_version():
     # primary resources other than .py files, this code will need
     # maintenance
     mismatch_py = [x for x in diffs.left_only + diffs.right_only + diffs.diff_files if x.endswith('.py')]
-    assert not mismatch_py, 'mismatched source files in %r and %r'% (pip_folder, pip_folder_outputed)
+    assert not mismatch_py, 'mismatched source files in %r and %r: %r'% (pip_folder, pip_folder_outputed, mismatch_py)
 
 
 def test_pip_second_command_line_interface_works():
@@ -302,6 +302,17 @@ def test_install_from_local_directory_with_no_setup_py():
     assert "is not installable. File 'setup.py' not found." in result.stdout
 
 
+def test_editable_install_from_local_directory_with_no_setup_py():
+    """
+    Test installing from a local directory with no 'setup.py'.
+    """
+    reset_env()
+    result = run_pip('install', '-e', here, expect_error=True)
+    assert len(result.files_created) == 1, result.files_created
+    assert 'pip-log.txt' in result.files_created, result.files_created
+    assert "is not installable. File 'setup.py' not found." in result.stdout
+
+
 def test_install_as_egg():
     """
     Test installing as egg, instead of flat install.
@@ -333,77 +344,6 @@ def test_install_curdir():
     assert egg_info_folder in result.files_created, str(result)
 
 
-def test_install_curdir_usersite_fails_in_old_python():
-    """
-    Test --user option on older Python versions (pre 2.6) fails intelligibly
-    """
-    if sys.version_info >= (2, 6):
-        raise SkipTest()
-    reset_env()
-    run_from = abspath(join(here, 'packages', 'FSPkg'))
-    result = run_pip('install', '--user', curdir, cwd=run_from, expect_error=True)
-    assert '--user is only supported in Python version 2.6 and newer' in result.stdout
-
-
-def test_install_curdir_usersite():
-    """
-    Test installing current directory ('.') into usersite
-    """
-    if sys.version_info < (2, 6):
-        raise SkipTest()
-    # FIXME distutils --user option seems to be broken in pypy
-    if hasattr(sys, "pypy_version_info"):
-        raise SkipTest()
-    env = reset_env(use_distribute=True)
-    run_from = abspath(join(here, 'packages', 'FSPkg'))
-    result = run_pip('install', '--user', curdir, cwd=run_from, expect_error=False)
-    fspkg_folder = env.user_site/'fspkg'
-    egg_info_folder = env.user_site/'FSPkg-0.1dev-py%s.egg-info' % pyversion
-    assert fspkg_folder in result.files_created, str(result.stdout)
-
-    assert egg_info_folder in result.files_created, str(result)
-
-
-def test_install_subversion_usersite_editable_with_distribute():
-    """
-    Test installing current directory ('.') into usersite after installing distribute
-    """
-    if sys.version_info < (2, 6):
-        raise SkipTest()
-    # FIXME distutils --user option seems to be broken in pypy
-    if hasattr(sys, "pypy_version_info"):
-        raise SkipTest()
-    env = reset_env(use_distribute=True)
-    (env.lib_path/'no-global-site-packages.txt').rm() # this one reenables user_site
-
-    result = run_pip('install', '--user', '-e',
-                     '%s#egg=initools-dev' %
-                     local_checkout('svn+http://svn.colorstudy.com/INITools/trunk'))
-    result.assert_installed('INITools', use_user_site=True)
-
-
-def test_install_subversion_usersite_editable_with_setuptools_fails():
-    """
-    Test installing current directory ('.') into usersite using setuptools fails
-    """
-    # --user only works on 2.6 or higher
-    if sys.version_info < (2, 6):
-        raise SkipTest()
-    # We don't try to use setuptools for 3.X.
-    elif sys.version_info >= (3,):
-        raise SkipTest()
-    env = reset_env(use_distribute=False)
-    no_site_packages = env.lib_path/'no-global-site-packages.txt'
-    if os.path.isfile(no_site_packages):
-        no_site_packages.rm() # this re-enables user_site
-
-    result = run_pip('install', '--user', '-e',
-                     '%s#egg=initools-dev' %
-                     local_checkout('svn+http://svn.colorstudy.com/INITools/trunk'),
-                     expect_error=True)
-    assert '--user --editable not supported with setuptools, use distribute' in result.stdout
-
-
 def test_install_pardir():
     """
     Test installing parent directory ('..').
@@ -615,3 +555,4 @@ def test_find_command_trys_supplied_pathext(mock_isfile, getpath_mock):
     assert_raises(BadCommand, find_command, 'foo', 'path_one', pathext)
     assert mock_isfile.call_args_list == expected, "Actual: %s\nExpected %s" % (mock_isfile.call_args_list, expected)
     assert not getpath_mock.called, "Should not call get_pathext"
+

tests/test_download.py

-from pip.backwardcompat import any
-
 import textwrap
 from tests.test_pip import reset_env, run_pip, write_file
 from tests.path import Path

tests/test_pip.py

@@ -102,13 +102,30 @@ def install_setuptools(env):
 env = None
 
 
-def reset_env(environ=None, use_distribute=None):
+def reset_env(environ=None, use_distribute=None, system_site_packages=False, sitecustomize=None):
+    """Return a test environment.
+
+    Keyword arguments:
+    environ: an environ object to use.
+    use_distribute: use distribute, not setuptools.
+    system_site_packages: create a virtualenv that simulates --system-site-packages.
+    sitecustomize: a string containing python code to add to sitecustomize.py.
+    """
+
     global env
     # FastTestPipEnv reuses env, not safe if use_distribute specified
-    if use_distribute is None:
-        env = FastTestPipEnvironment(environ)
+    if use_distribute is None and not system_site_packages:
+        env = FastTestPipEnvironment(environ, sitecustomize=sitecustomize)
     else:
-        env = TestPipEnvironment(environ, use_distribute=use_distribute)
+        env = TestPipEnvironment(environ, use_distribute=use_distribute, sitecustomize=sitecustomize)
+
+    if system_site_packages:
+        #testing often occurs starting from a private virtualenv (e.g. with tox)
+        #from that context, you can't successfully use virtualenv.create_environment
+        #to create a 'system-site-packages' virtualenv
+        #hence, this workaround
+        (env.lib_path/'no-global-site-packages.txt').rm()
+
     return env
 
 
@@ -262,7 +279,7 @@ class TestPipEnvironment(TestFileEnvironment):
 
     verbose = False
 
-    def __init__(self, environ=None, use_distribute=None):
+    def __init__(self, environ=None, use_distribute=None, sitecustomize=None):
 
         self.root_path = Path(tempfile.mkdtemp('-piptest'))
 
@@ -338,7 +355,12 @@ class TestPipEnvironment(TestFileEnvironment):
 
         # Install this version instead
         self.run('python', 'setup.py', 'install', cwd=src_folder, expect_stderr=True)
+
+        #create sitecustomize.py and add patches
+        self._create_empty_sitecustomize()
         self._use_cached_pypi_server()
+        if sitecustomize:
+            self._add_to_sitecustomize(sitecustomize)
 
     def _ignore_file(self, fn):
         if fn.endswith('__pycache__') or fn.endswith(".pyc"):
@@ -361,21 +383,41 @@ class TestPipEnvironment(TestFileEnvironment):
         rmtree(str(self.root_path), ignore_errors=True)
 
     def _use_cached_pypi_server(self):
-        site_packages = self.root_path / self.site_packages
-        pth = open(os.path.join(site_packages, 'pypi_intercept.pth'), 'w')
-        pth.write('import sys; ')
-        pth.write('sys.path.insert(0, %r); ' % str(here))
-        pth.write('import pypi_server; pypi_server.PyPIProxy.setup(); ')
-        pth.write('sys.path.remove(%r); ' % str(here))
-        pth.close()
-
+        # previously, this was handled in a pth file, and not in sitecustomize.py
+        # pth processing happens during the construction of sys.path.
+        # 'import pypi_server' ultimately imports pkg_resources (which intializes pkg_resources.working_set based on the current state of sys.path)
+        # pkg_resources.get_distribution (used in pip.req) requires an accurate pkg_resources.working_set
+        # therefore, 'import pypi_server' shouldn't occur in a pth file.
+
+        patch = """
+            import sys
+            sys.path.insert(0, %r)
+            import pypi_server
+            pypi_server.PyPIProxy.setup()
+            sys.path.remove(%r)""" % (str(here), str(here))
+        self._add_to_sitecustomize(patch)
+
+    def _create_empty_sitecustomize(self):
+        "Create empty sitecustomize.py."
+        sitecustomize_path = self.lib_path / 'sitecustomize.py'
+        sitecustomize = open(sitecustomize_path, 'w')
+        sitecustomize.close()
+
+    def _add_to_sitecustomize(self, snippet):
+        "Adds a python code snippet to sitecustomize.py."
+        sitecustomize_path = self.lib_path / 'sitecustomize.py'
+        sitecustomize = open(sitecustomize_path, 'a')
+        sitecustomize.write(textwrap.dedent('''
+                               %s
+        ''' %snippet))
+        sitecustomize.close()
 
 fast_test_env_root = here / 'tests_cache' / 'test_ws'
 fast_test_env_backup = here / 'tests_cache' / 'test_ws_backup'
 
 
 class FastTestPipEnvironment(TestPipEnvironment):
-    def __init__(self, environ=None):
+    def __init__(self, environ=None, sitecustomize=None):
         import virtualenv
 
         self.root_path = fast_test_env_root
@@ -459,7 +501,13 @@ class FastTestPipEnvironment(TestPipEnvironment):
             # Install this version instead
             self.run('python', 'setup.py', 'install', cwd=src_folder, expect_stderr=True)
             shutil.copytree(self.root_path, self.backup_path, True)
+
+        #create sitecustomize.py and add patches
+        self._create_empty_sitecustomize()
         self._use_cached_pypi_server()
+        if sitecustomize:
+            self._add_to_sitecustomize(sitecustomize)
+
         assert self.root_path.exists
 
     def __del__(self):

tests/test_requirements.py

@@ -121,17 +121,19 @@ def test_requirements_data_structure_implements__contains__():
     assert 'pip' in requirements
     assert 'nose' not in requirements
 
+@patch('pip.req.os.getcwd')
 @patch('pip.req.os.path.exists')
 @patch('pip.req.os.path.isdir')
-def test_parse_editable_local(isdir_mock, exists_mock):
+def test_parse_editable_local(isdir_mock, exists_mock, getcwd_mock):
     exists_mock.return_value = isdir_mock.return_value = True
+    getcwd_mock.return_value = "/some/path"
     assert_equal(
         parse_editable('.', 'git'),
-        (None, 'file://' + os.getcwd(), None)
+        (None, 'file:///some/path', None)
     )
     assert_equal(
         parse_editable('foo', 'git'),
-        (None, 'file://' + os.path.join(os.getcwd(), 'foo'), None)
+        (None, 'file://' + os.path.join("/some/path", 'foo'), None)
     )
 
 def test_parse_editable_default_vcs():
@@ -152,17 +154,19 @@ def test_parse_editable_vcs_extras():
         ('foo[extras]', 'svn+https://foo#egg=foo[extras]', None)
     )
 
+@patch('pip.req.os.getcwd')
 @patch('pip.req.os.path.exists')
 @patch('pip.req.os.path.isdir')
-def test_parse_editable_local_extras(isdir_mock, exists_mock):
+def test_parse_editable_local_extras(isdir_mock, exists_mock, getcwd_mock):
     exists_mock.return_value = isdir_mock.return_value = True
+    getcwd_mock.return_value = "/some/path"
     assert_equal(
         parse_editable('.[extras]', 'git'),
-        (None, 'file://' + os.getcwd(), ('extras',))
+        (None, 'file://' + "/some/path", ('extras',))
     )
     assert_equal(
         parse_editable('foo[bar,baz]', 'git'),
-        (None, 'file://' + os.path.join(os.getcwd(), 'foo'), ('bar', 'baz'))
+        (None, 'file://' + os.path.join("/some/path", 'foo'), ('bar', 'baz'))
     )
 
 def test_install_local_editable_with_extras():

tests/test_test.py

+"""Test the test support."""
+
+import sys
+import os
+from os.path import abspath, join, curdir, isdir, isfile
+from nose import SkipTest
+from tests.local_repos import local_checkout
+from tests.test_pip import here, reset_env, run_pip, pyversion
+
+
+patch_urlopen = """
+       def mock_urlopen():
+           pass
+       import pip
+       pip.backwardcompat.urllib2.urlopen = mock_urlopen
+    """
+
+def test_pypiproxy_patch_applied():
+    """
+    Test the PyPIProxy.setup() patch was applied, and sys.path returned to normal
+    """
+
+    env = reset_env()
+    result = env.run('python', '-c', "import pip; print(pip.backwardcompat.urllib2.urlopen.__module__)")
+    #if it were not patched, the result would be 'urllib2'
+    assert "pypi_server"== result.stdout.strip(), result.stdout
+
+    #confirm the temporary sys.path adjustment is gone
+    result = env.run('python', '-c', "import sys; print(sys.path)")
+    paths = eval(result.stdout.strip())
+    assert here not in paths, paths
+
+
+def test_add_patch_to_sitecustomize():
+    """
+    Test adding monkey patch snippet to sitecustomize.py (using TestPipEnvironment)
+    """
+
+    env = reset_env(sitecustomize=patch_urlopen, use_distribute=True)
+    result = env.run('python', '-c', "import pip; print(pip.backwardcompat.urllib2.urlopen.__module__)")
+    assert "sitecustomize"== result.stdout.strip(), result.stdout
+
+
+def test_add_patch_to_sitecustomize_fast():
+    """
+    Test adding monkey patch snippet to sitecustomize.py (using FastTestPipEnvironment)
+    """
+
+    env = reset_env(sitecustomize=patch_urlopen)
+    result = env.run('python', '-c', "import pip; print(pip.backwardcompat.urllib2.urlopen.__module__)")
+    assert "sitecustomize"== result.stdout.strip(), result.stdout
+
+
+def test_sitecustomize_not_growing_in_fast_environment():
+    """
+    Test that the sitecustomize is not growing with redundant patches in the cached fast environment
+    """
+
+    patch = "fu = 'bar'"
+
+    env1 = reset_env(sitecustomize=patch)
+    sc1 = env1.lib_path / 'sitecustomize.py'
+    size1 = os.stat(sc1).st_size
+    env2 = reset_env(sitecustomize=patch)
+    sc2 = env2.lib_path / 'sitecustomize.py'
+    size2 = os.stat(sc2).st_size
+    assert size1==size2, "size before, %d != size after, %d" %(size1, size2)
+

tests/test_uninstall.py

@@ -14,9 +14,9 @@ def test_simple_uninstall():
 
     """
     env = reset_env()
-    result = run_pip('install', 'INITools==0.2', expect_error=True)
+    result = run_pip('install', 'INITools==0.2')
     assert join(env.site_packages, 'initools') in result.files_created, sorted(result.files_created.keys())
-    result2 = run_pip('uninstall', 'INITools', '-y', expect_error=True)
+    result2 = run_pip('uninstall', 'INITools', '-y')
     assert_all_changes(result, result2, [env.venv/'build', 'cache'])
 
 

tests/test_upgrade.py

 import textwrap
 from os.path import join
+from nose.tools import nottest
 from tests.test_pip import (here, reset_env, run_pip, assert_all_changes,
                             write_file, pyversion, _create_test_package,
                             _change_test_package_version)
@@ -150,7 +151,8 @@ def test_uninstall_rollback():
     assert env.run('python', '-c', "import broken; print(broken.VERSION)").stdout == '0.1\n'
     assert_all_changes(result.files_after, result2, [env.venv/'build', 'pip-log.txt'])
 
-
+# Issue #530 - temporarily disable flaky test
+@nottest
 def test_editable_git_upgrade():
     """
     Test installing an editable git package from a repository, upgrading the repository,

tests/test_user_site.py

+"""
+tests specific to "--user" option
+"""
+
+import sys
+from os.path import abspath, join, curdir, isdir, isfile
+from nose import SkipTest
+from tests.local_repos import local_checkout
+from tests.test_pip import here, reset_env, run_pip, pyversion
+
+
+def test_install_curdir_usersite_fails_in_old_python():
+    """
+    Test --user option on older Python versions (pre 2.6) fails intelligibly
+    """
+    if sys.version_info >= (2, 6):
+        raise SkipTest()
+    reset_env(system_site_packages=True)
+    run_from = abspath(join(here, 'packages', 'FSPkg'))
+    result = run_pip('install', '--user', curdir, cwd=run_from, expect_error=True)
+    assert '--user is only supported in Python version 2.6 and newer' in result.stdout
+
+
+class Tests_UserSite:
+
+    def setup(self):
+        # --user only works on 2.6 or higher
+        if sys.version_info < (2, 6):
+            raise SkipTest()
+
+    def test_reset_env_system_site_packages_usersite(self):
+        """
+        reset_env(system_site_packages=True) produces env where a --user install can be found using pkg_resources
+        """
+        env = reset_env(system_site_packages=True)
+        run_pip('install', '--user', 'INITools==0.2')
+        result = env.run('python', '-c', "import pkg_resources; print(pkg_resources.get_distribution('initools').project_name)")
+        project_name = result.stdout.strip()
+        assert 'INITools'== project_name, "'%s' should be 'INITools'" %project_name
+
+
+    def test_install_subversion_usersite_editable_with_setuptools_fails(self):
+        """
+        Test installing current directory ('.') into usersite using setuptools fails
+        """
+        # We don't try to use setuptools for 3.X.
+        if sys.version_info >= (3,):
+            raise SkipTest()
+        env = reset_env(use_distribute=False, system_site_packages=True)
+        result = run_pip('install', '--user', '-e',
+                         '%s#egg=initools-dev' %
+                         local_checkout('svn+http://svn.colorstudy.com/INITools/trunk'),
+                         expect_error=True)
+        assert '--user --editable not supported with setuptools, use distribute' in result.stdout
+
+
+    def test_install_subversion_usersite_editable_with_distribute(self):
+        """
+        Test installing current directory ('.') into usersite after installing distribute
+        """
+        # FIXME distutils --user option seems to be broken in pypy
+        if hasattr(sys, "pypy_version_info"):
+            raise SkipTest()
+        env = reset_env(use_distribute=True, system_site_packages=True)
+        result = run_pip('install', '--user', '-e',
+                         '%s#egg=initools-dev' %
+                         local_checkout('svn+http://svn.colorstudy.com/INITools/trunk'))
+        result.assert_installed('INITools', use_user_site=True)
+
+
+    def test_install_curdir_usersite(self):
+        """
+        Test installing current directory ('.') into usersite
+        """
+        # FIXME distutils --user option seems to be broken in pypy
+        if hasattr(sys, "pypy_version_info"):
+            raise SkipTest()
+        env = reset_env(use_distribute=True, system_site_packages=True)
+        run_from = abspath(join(here, 'packages', 'FSPkg'))
+        result = run_pip('install', '--user', curdir, cwd=run_from, expect_error=False)
+        fspkg_folder = env.user_site/'fspkg'
+        egg_info_folder = env.user_site/'FSPkg-0.1dev-py%s.egg-info' % pyversion
+        assert fspkg_folder in result.files_created, str(result.stdout)
+
+        assert egg_info_folder in result.files_created, str(result)
+
+
+    def test_install_user_venv_nositepkgs_fails(self):
+        """
+        user install in virtualenv (with no system packages) fails with message
+        """
+        env = reset_env()
+        run_from = abspath(join(here, 'packages', 'FSPkg'))
+        result = run_pip('install', '--user', curdir, cwd=run_from, expect_error=True)
+        assert "Can not perform a '--user' install. User site-packages are not visible in this virtualenv." in result.stdout
+
+
+    def test_install_user_conflict_in_usersite(self):
+        """
+        Test user install with conflict in usersite updates usersite.
+        """
+
+        env = reset_env(system_site_packages=True)
+        result1 = run_pip('install', '--user', 'INITools==0.3')
+        result2 = run_pip('install', '--user', 'INITools==0.1')
+
+        #usersite has 0.1
+        egg_info_folder = env.user_site / 'INITools-0.1-py%s.egg-info' % pyversion
+        initools_v3_file = env.root_path / env.user_site / 'initools' / 'configparser.py' #file only in 0.3
+        assert egg_info_folder in result2.files_created, str(result2)
+        assert not isfile(initools_v3_file), initools_v3_file
+
+
+    def test_install_user_conflict_in_site(self):
+        """
+        Test user install with conflict in site ignores site and installs to usersite
+        """
+
+        #the test framework only supports testing using virtualenvs
+        #this test will use a --system_site_packages virtualenv to achieve the conflict scenario.
+
+        env = reset_env(system_site_packages=True)
+        result1 = run_pip('install', 'INITools==0.2')
+        result2 = run_pip('install', '--user', 'INITools==0.1')
+
+        #usersite has 0.1
+        egg_info_folder = env.user_site / 'INITools-0.1-py%s.egg-info' % pyversion
+        initools_folder = env.user_site / 'initools'
+        assert egg_info_folder in result2.files_created, str(result2)
+        assert initools_folder in result2.files_created, str(result2)
+
+        #site still has 0.2 (can't look in result1; have to check)
+        egg_info_folder = env.root_path / env.site_packages / 'INITools-0.2-py%s.egg-info' % pyversion
+        initools_folder = env.root_path / env.site_packages / 'initools'
+        assert isdir(egg_info_folder)
+        assert isdir(initools_folder)
+
+
+    def test_install_user_conflict_in_globalsite_and_usersite(self):
+        """
+        Test user install with conflict in globalsite and usersite ignores global site and updates usersite.
+        """
+
+        #the test framework only supports testing using virtualenvs
+        #this test will use a --system_site_packages virtualenv to achieve the conflict scenario.
+
+        env = reset_env(system_site_packages=True)
+
+        # the sys.path ordering for virtualenvs with --system-site-packages is this: virtualenv site, usersite, global site
+        # given this ordering you *can't* use it to simulate the scenario for this test.
+        # this test will add the usersite to PYTHONPATH to simulate the desired ordering
+        env.environ["PYTHONPATH"] = env.root_path / env.user_site
+
+        result1 = run_pip('install', 'INITools==0.2')
+        result2 = run_pip('install', '--user', 'INITools==0.3')
+        result3 = run_pip('install', '--user', 'INITools==0.1')
+
+        #usersite has 0.1
+        egg_info_folder = env.user_site / 'INITools-0.1-py%s.egg-info' % pyversion
+        initools_v3_file = env.root_path / env.user_site / 'initools' / 'configparser.py'  #file only in 0.3
+        assert egg_info_folder in result3.files_created, str(result3)
+        assert not isfile(initools_v3_file), initools_v3_file
+
+        #site still has 0.2 (can't just look in result1; have to check)
+        egg_info_folder = env.root_path / env.site_packages / 'INITools-0.2-py%s.egg-info' % pyversion
+        initools_folder = env.root_path / env.site_packages / 'initools'
+        assert isdir(egg_info_folder)
+        assert isdir(initools_folder)

tests/test_vcs_backends.py

@@ -67,7 +67,7 @@ def test_git_with_tag_name_and_update():
                      '%s@0.1.2#egg=pip-test-package' %
                      local_checkout('git+http://github.com/pypa/pip-test-package.git'),
                      expect_error=True)
-    assert '0.1.2\n' in result.stdout
+    assert '0.1.2' in result.stdout
 
 
 def test_git_branch_should_not_be_changed():
@@ -92,7 +92,7 @@ def test_git_with_non_editable_unpacking():
     result = run_pip('install', '--global-option=--version', local_checkout(
                      'git+http://github.com/pypa/pip-test-package.git@0.1.2#egg=pip-test-package'
                      ), expect_error=True)
-    assert '0.1.2\n' in result.stdout
+    assert '0.1.2' in result.stdout
 
 
 def test_git_with_editable_where_egg_contains_dev_string():

tests/test_vcs_git.py

 from mock import patch
 from pip.vcs.git import Git
 from tests.test_pip import (reset_env, run_pip,
-                            _create_test_package)
+                            _create_test_package,)
+from tests.git_submodule_helpers import (
+    _change_test_package_submodule,
+    _pull_in_submodule_changes_to_module,
+    _create_test_package_with_submodule,
+    )
 
 
 def test_get_tag_revs_should_return_tag_name_and_commit_pair():
@@ -75,3 +80,24 @@ def test_check_rev_options_should_handle_ambiguous_commit(branches_revs_mock,
 
     result = git.check_rev_options('0.1', '.', [])
     assert result == ['123456'], result
+
+
+def test_check_submodule_addition():
+    """
+    Submodules are pulled in on install and updated on upgrade.
+
+    """
+    env = reset_env()
+    module_path, submodule_path = _create_test_package_with_submodule(env)
+
+    install_result = run_pip('install', '-e', 'git+'+module_path+'#egg=version_pkg')
+    assert '.virtualenv/src/version-pkg/testpkg/static/testfile' in install_result.files_created
+
+    _change_test_package_submodule(env, submodule_path)
+    _pull_in_submodule_changes_to_module(env, module_path)
+
+    # expect error because git may write to stderr
+    update_result = run_pip('install', '-e', 'git+'+module_path+'#egg=version_pkg', '--upgrade', expect_error=True)
+
+    assert env.venv/'src/version-pkg/testpkg/static/testfile2' in update_result.files_created
+