Merge pull request #4835 from pradyunsg/vendoring/enable-c-libs

Enable vendor C libraries on non-Windows platforms

news/4098.bugfix

+Limit the disabling of requests' pyopenssl to Windows only. Fixes "SNIMissingWarning / InsecurePlatformWarning not fixable with pip 9.0 / 9.0.1" (for non-Windows)

src/pip/_internal/download.py

@@ -27,8 +27,10 @@ from pip._vendor.six.moves import xmlrpc_client  # type: ignore
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
 from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
+from pip._vendor.urllib3.util import IS_PYOPENSSL
 
 import pip
+from pip._internal.compat import WINDOWS
 from pip._internal.exceptions import HashMismatch, InstallationError
 from pip._internal.locations import write_delete_marker_file
 from pip._internal.models import PyPI
@@ -48,9 +50,10 @@ from pip._internal.vcs import vcs
 
 try:
     import ssl  # noqa
-    HAS_TLS = True
 except ImportError:
-    HAS_TLS = False
+    ssl = None
+
+HAS_TLS = (ssl is not None) or IS_PYOPENSSL
 
 __all__ = ['get_file_content',
            'is_url', 'url_to_path', 'path_to_url',

src/pip/_vendor/README.rst

@@ -43,30 +43,30 @@ way (via ``install_requires``) for pip. These issues are:
 
 * **Making other libraries uninstallable.** One of pip's current dependencies is
   the ``requests`` library, for which pip requires a fairly recent version to run.
-  If pip dependended on ``requests`` in the traditional manner, then we'd either 
-  have to maintain compatibility with every ``requests`` version that has ever 
+  If pip dependended on ``requests`` in the traditional manner, then we'd either
+  have to maintain compatibility with every ``requests`` version that has ever
   existed (and ever will), OR allow pip to render certain versions of ``requests``
-  uninstallable. (The second issue, although technically true for any Python 
-  application, is magnified by pip's ubiquity; pip is installed by default in 
+  uninstallable. (The second issue, although technically true for any Python
+  application, is magnified by pip's ubiquity; pip is installed by default in
   Python, in ``pyvenv``, and in ``virtualenv``.)
 
-* **Security.** This might seem puzzling at first glance, since vendoring 
+* **Security.** This might seem puzzling at first glance, since vendoring
   has a tendency to complicate updating dependencies for security updates,
-  and that holds true for pip. However, given the *other* reasons for avoiding 
-  dependencies, the alternative is for pip to reinvent the wheel itself. 
-  This is what pip did historically. It forced pip to re-implement its own 
-  HTTPS verification routines as a workaround for the Python standard library's 
-  lack of SSL validation, which resulted in similar bugs in the validation routine 
+  and that holds true for pip. However, given the *other* reasons for avoiding
+  dependencies, the alternative is for pip to reinvent the wheel itself.
+  This is what pip did historically. It forced pip to re-implement its own
+  HTTPS verification routines as a workaround for the Python standard library's
+  lack of SSL validation, which resulted in similar bugs in the validation routine
   in ``requests`` and ``urllib3``, except that they had to be discovered and
-  fixed independently. Even though we're vendoring, reusing libraries keeps pip 
+  fixed independently. Even though we're vendoring, reusing libraries keeps pip
   more secure by relying on the great work of our dependencies, *and* allowing for
   faster, easier security fixes by simply pulling in newer versions of dependencies.
 
 * **Bootstrapping.** Currently most popular methods of installing pip rely
-  on pip's self-contained nature to install pip itself. These tools work by bundling 
-  a copy of pip, adding it to ``sys.path``, and then executing that copy of pip. 
-  This is done instead of implementing a "mini installer" (to reduce duplication); 
-  pip already knows how to install a Python package, and is far more battle-tested 
+  on pip's self-contained nature to install pip itself. These tools work by bundling
+  a copy of pip, adding it to ``sys.path``, and then executing that copy of pip.
+  This is done instead of implementing a "mini installer" (to reduce duplication);
+  pip already knows how to install a Python package, and is far more battle-tested
   than any "mini installer" could ever possibly be.
 
 Many downstream redistributors have policies against this kind of bundling, and
@@ -92,12 +92,13 @@ such as OS packages.
 Modifications
 -------------
 
-* ``html5lib`` has been modified to ``import six from pip._vendor``
 * ``setuptools`` is completely stripped to only keep ``pkg_resources``
 * ``pkg_resources`` has been modified to import its dependencies from ``pip._vendor``
-* ``CacheControl`` has been modified to import its dependencies from ``pip._vendor``
 * ``packaging`` has been modified to import its dependencies from ``pip._vendor``
-* ``requests`` has been modified *not* to optionally load any C dependencies
+* ``html5lib`` has been modified to ``import six from pip._vendor``
+* ``CacheControl`` has been modified to import its dependencies from ``pip._vendor``
+* ``requests`` has been modified to import its other dependencies from ``pip._vendor``
+  and to *not* load ``simplejson`` (all platforms) and ``pyopenssl`` (Windows).
 
 
 Automatic Vendoring

src/pip/_vendor/requests/__init__.py

@@ -80,13 +80,13 @@ except (AssertionError, ValueError):
                   RequestsDependencyWarning)
 
 # Attempt to enable urllib3's SNI support, if possible
-# Note: Patched by pip to prevent using the PyOpenSSL module. On Windows this
-#       prevents upgrading cryptography.
-# try:
-#     from pip._vendor.urllib3.contrib import pyopenssl
-#     pyopenssl.inject_into_urllib3()
-# except ImportError:
-#     pass
+from pip._internal.compat import WINDOWS
+if not WINDOWS:
+    try:
+        from pip._vendor.urllib3.contrib import pyopenssl
+        pyopenssl.inject_into_urllib3()
+    except ImportError:
+        pass
 
 # urllib3's DependencyWarnings should be silenced.
 from pip._vendor.urllib3.exceptions import DependencyWarning

tasks/vendoring/patches/requests.patch

@@ -21,10 +21,10 @@ index 6336a07d..9582fa73 100644
  # Kinda cool, though, right?
 
 diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py
-index 9c3b769..44f6836 100644
+index 9c3b769..36a4ef40 100644
 --- a/src/pip/_vendor/requests/__init__.py
 +++ b/src/pip/_vendor/requests/__init__.py
-@@ -80,11 +80,13 @@
+@@ -80,10 +80,12 @@ except (AssertionError, ValueError):
                    RequestsDependencyWarning)
 
  # Attempt to enable urllib3's SNI support, if possible
@@ -33,17 +33,15 @@ index 9c3b769..44f6836 100644
 -    pyopenssl.inject_into_urllib3()
 -except ImportError:
 -    pass
-+# Note: Patched by pip to prevent using the PyOpenSSL module. On Windows this
-+#       prevents upgrading cryptography.
-+# try:
-+#     from pip._vendor.urllib3.contrib import pyopenssl
-+#     pyopenssl.inject_into_urllib3()
-+# except ImportError:
-+#     pass
++from pip._internal.compat import WINDOWS
++if not WINDOWS:
++    try:
++        from pip._vendor.urllib3.contrib import pyopenssl
++        pyopenssl.inject_into_urllib3()
++    except ImportError:
++        pass
 
  # urllib3's DependencyWarnings should be silenced.
- from pip._vendor.urllib3.exceptions import DependencyWarning
- 
 diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py
 index eb6530d..353ec29 100644
 --- a/src/pip/_vendor/requests/compat.py