From e80c387a26858c4d7ff43c5f030b04b03fd43dfe Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 16 Aug 2010 01:46:23 +0200 Subject: [PATCH] Added support for mirrors as defined in PEP 381. This feature is disabled by default and will query the DNS entry of the main mirror index URL to get a list of mirrors. Optinally can be passed a list of mirrors instead. --- docs/index.txt | 21 +++++++++++ docs/news.txt | 3 ++ pip/backwardcompat.py | 9 +++++ pip/commands/install.py | 20 +++++++++- pip/index.py | 82 ++++++++++++++++++++++++++++++++++++++--- tests/test_basic.py | 24 ++++++++++++ 6 files changed, 152 insertions(+), 7 deletions(-) diff --git a/docs/index.txt b/docs/index.txt index a5aa0373c..43cc9da6d 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -316,3 +316,24 @@ indexed. pip searches http://pypi.python.org/pypi by default but alternative indexes can be searched by using the ``--index`` flag. + +Mirror support +-------------- + +The `PyPI mirroring infrastructure `_ as +described in `PEP 381 `_ can be +used by passing the ``--use-mirrors`` option to the install command. +Alternatively, you can use the other ways to configure pip, e.g.:: + + $ export PIP_USE_MIRRORS=true + +If enabled, pip will automatically query the DNS entry of the mirror index URL +to find the list of mirrors to use. In case you want to override this list, +please use the ``--mirrors`` option of the install command, or add to your pip +configuration file:: + + [install] + use-mirrors = true + mirrors = + http://d.pypi.python.org + http://b.pypi.python.org diff --git a/docs/news.txt b/docs/news.txt index dd99b9165..5d181b1df 100644 --- a/docs/news.txt +++ b/docs/news.txt @@ -4,6 +4,9 @@ News for pip tip --- +* Added support for `PyPI mirrors `_ as + defined in `PEP 381 `_, from + Jannis Leidel. 0.8 --- diff --git a/pip/backwardcompat.py b/pip/backwardcompat.py index cd7ab88ff..e7c11f1d3 100644 --- a/pip/backwardcompat.py +++ b/pip/backwardcompat.py @@ -44,3 +44,12 @@ def copytree(src, dst): shutil.copytree(src, dst) +def product(*args, **kwds): + # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) diff --git a/pip/commands/install.py b/pip/commands/install.py index 8c499763f..827fe86cd 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -45,7 +45,7 @@ class InstallCommand(Command): '-i', '--index-url', '--pypi-url', dest='index_url', metavar='URL', - default='http://pypi.python.org/simple', + default='http://pypi.python.org/simple/', help='Base URL of Python Package Index (default %default)') self.parser.add_option( '--extra-index-url', @@ -60,6 +60,19 @@ class InstallCommand(Command): action='store_true', default=False, help='Ignore package index (only looking at --find-links URLs instead)') + self.parser.add_option( + '-M', '--use-mirrors', + dest='use_mirrors', + action='store_true', + default=False, + help='Use the PyPI mirrors as a fallback in case the main index is down.') + self.parser.add_option( + '--mirrors', + dest='mirrors', + metavar='URL', + action='append', + default=[], + help='Specific mirror URLs to query when --use-mirrors is used') self.parser.add_option( '-b', '--build', '--build-dir', '--build-directory', @@ -136,7 +149,10 @@ class InstallCommand(Command): This method is meant to be overridden by subclasses, not called directly. """ - return PackageFinder(find_links=options.find_links, index_urls=index_urls) + return PackageFinder(find_links=options.find_links, + index_urls=index_urls, + use_mirrors=options.use_mirrors, + mirrors=options.mirrors) def run(self, options, args): if not options.build_dir: diff --git a/pip/index.py b/pip/index.py index 4c82a45bf..c0e37251a 100644 --- a/pip/index.py +++ b/pip/index.py @@ -11,19 +11,24 @@ import urllib import urllib2 import urlparse import httplib +import random import socket +import string from Queue import Queue from Queue import Empty as QueueEmpty from pip.log import logger from pip.util import Inf from pip.util import normalize_name, splitext from pip.exceptions import DistributionNotFound -from pip.backwardcompat import WindowsError +from pip.backwardcompat import WindowsError, product from pip.download import urlopen, path_to_url2, url_to_path, geturl __all__ = ['PackageFinder'] +DEFAULT_MIRROR_URL = "last.pypi.python.org" + + class PackageFinder(object): """This finds packages. @@ -31,15 +36,19 @@ class PackageFinder(object): packages, by reading pages and looking for appropriate links """ - failure_limit = 3 - - def __init__(self, find_links, index_urls): + def __init__(self, find_links, index_urls, + use_mirrors=False, mirrors=None, main_mirror_url=None): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] self.cache = PageCache() # These are boring links that have already been logged somehow: self.logged_links = set() + if use_mirrors: + self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) + logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) + else: + self.mirror_urls = [] def add_dependency_links(self, links): ## FIXME: this shouldn't be global list this, it should only @@ -91,6 +100,10 @@ class PackageFinder(object): if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name + # Combine index URLs with mirror URLs here to allow + # adding more index URLs from requirements files + all_index_urls = self.index_urls + self.mirror_urls + def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path @@ -103,7 +116,7 @@ class PackageFinder(object): if url_name is not None: locations = [ mkurl_pypi_url(url) - for url in self.index_urls] + self.find_links + for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) @@ -312,6 +325,26 @@ class PackageFinder(object): def _get_page(self, link, req): return HTMLPage.get_page(link, req, cache=self.cache) + def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): + """Retrieves a list of URLs from the main mirror DNS entry + unless a list of mirror URLs are passed. + """ + if not mirrors: + mirrors = get_mirrors(main_mirror_url) + # Should this be made "less random"? E.g. netselect like? + random.shuffle(mirrors) + + mirror_urls = set() + for mirror_url in mirrors: + # Make sure we have a valid URL + if not ("http://" or "https://" or "file://") in mirror_url: + mirror_url = "http://%s" % mirror_url + if not mirror_url.endswith("/simple"): + mirror_url = "%s/simple/" % mirror_url + mirror_urls.add(mirror_url) + + return list(mirror_urls) + class PageCache(object): """Cache of HTML pages""" @@ -619,3 +652,42 @@ def package_to_requirement(package_name): return '%s==%s' % (name, version) else: return name + + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from pip.index import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + Originally written for the distutils2 project by Alexis Metaireau. + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(string.ascii_lowercase, repeat=k+1): + result = ''.join(x) + yield result + if result == last: + return + diff --git a/tests/test_basic.py b/tests/test_basic.py index 2a0321160..08b5c5f3d 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -71,6 +71,30 @@ def test_install_from_pypi(): assert initools_folder in result.files_created, str(result) +def test_install_from_mirrors(): + """ + Test installing a package from the PyPI mirrors. + """ + e = reset_env() + result = run_pip('install', '-vvv', '--use-mirrors', '--no-index', 'INITools==0.2') + egg_info_folder = e.site_packages / 'INITools-0.2-py%s.egg-info' % pyversion + initools_folder = e.site_packages / 'initools' + assert egg_info_folder in result.files_created, str(result) + assert initools_folder in result.files_created, str(result) + + +def test_install_from_mirrors_with_specific_mirrors(): + """ + Test installing a package from a specific PyPI mirror. + """ + e = reset_env() + result = run_pip('install', '-vvv', '--use-mirrors', '--mirrors', "http://d.pypi.python.org/", '--no-index', 'INITools==0.2') + egg_info_folder = e.site_packages / 'INITools-0.2-py%s.egg-info' % pyversion + initools_folder = e.site_packages / 'initools' + assert egg_info_folder in result.files_created, str(result) + assert initools_folder in result.files_created, str(result) + + def test_editable_install(): """ Test editable installation. -- GitLab