Merge pull request #186 from frostming/bugfix/185

change the mechanism to enable PEP 582

README.md

@@ -108,7 +108,7 @@ if __name__ == '__main__':
     app.run()
 ```
 
-Set environment variable `export PDM_PYTHON_PEP582=1`. Now you can run the app directly with your familiar **Python interpreter**:
+Set environment variable `eval $(pdm --pep582)`. Now you can run the app directly with your familiar **Python interpreter**:
 
 ```bash
 $ python /home/frostming/workspace/flask_app/app.py

docs/docs/index.md

@@ -46,11 +46,15 @@ $ pip install --user pdm
 
 ### Enable PEP 582 globally
 
-To make the Python interpreters aware of PEP 582 packages, set the environment variable `PDM_PYTHON_PEP582` to `1`.
-You may want to write a line in your `.bash_profile`(or similar profiles) to make it effective when login:
+To make the Python interpreters aware of PEP 582 packages, one need to add the `pdm/pep582/sitecustomize.py`
+to the Python library search path. The command can be produced by `pdm --pep582 [<SHELL>]` and if `<SHELL>`
+isn't given, PDM will pick one based on some guesses.
+
+You may want to write a line in your `.bash_profile`(or similar profiles) to make it effective when login.
+For example, in bash you can do this:
 
 ```bash
-export PDM_PYTHON_PEP582=1
+$ pdm --pep582 >> ~/.bash_profile
 ```
 
 **This setup may become the default in the future.**

docs/docs/project.md

@@ -287,5 +287,5 @@ By default, system-level site-packages will be excluded from the `sys.path` when
 ## How we make PEP 582 packages available to the Python interpreter
 
 Thanks to the [site packages loading](https://docs.python.org/3/library/site.html) on Python startup. It is possible to patch the `sys.path`
-by placing a `_pdm_pep582.pth` together with a small script under the `site-packages` directory. The interpreter can search the directories
-for the neareset `__pypackage__` folder and append it to the `sys.path` variable. This is totally done by PDM and users shouldn't be aware.
+by executing the `sitecustomize.py` shipped with PDM. The interpreter can search the directories
+for the neareset `__pypackage__` folder and append it to the `sys.path` variable.

news/185.bugfix

+Write a `sitecustomize.py` instead of a `.pth` file to enable PEP 582.
+Update `get_package_finder()` to be compatible with `pip 20.3`.

pdm.lock

@@ -268,7 +268,7 @@ zipp = {marker = "python_version < '3.8'", version = "*"}
 [[package]]
 name = "pip"
 sections = ["default"]
-version = "20.2.4"
+version = "20.3.1"
 summary = "The PyPA recommended tool for installing Python packages."
 
 [[package]]
@@ -457,6 +457,12 @@ sections = ["default", "dev"]
 version = "50.3.2"
 summary = "Easily download, build, install, upgrade, and uninstall Python packages"
 
+[[package]]
+name = "shellingham"
+sections = ["default"]
+version = "1.3.2"
+summary = "Tool to Detect Surrounding Shell"
+
 [[package]]
 name = "six"
 sections = ["default", "dev", "doc"]
@@ -722,9 +728,9 @@ summary = "Backport of pathlib-compatible object wrapper for zip files"
     {file = "pep517-0.9.1-py2.py3-none-any.whl", hash = "sha256:3985b91ebf576883efe5fa501f42a16de2607684f3797ddba7202b71b7d0da51"},
     {file = "pep517-0.9.1.tar.gz", hash = "sha256:aeb78601f2d1aa461960b43add204cc7955667687fbcf9cdb5170f00556f117f"},
 ]
-"pip 20.2.4" = [
-    {file = "pip-20.2.4-py2.py3-none-any.whl", hash = "sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033"},
-    {file = "pip-20.2.4.tar.gz", hash = "sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1"},
+"pip 20.3.1" = [
+    {file = "pip-20.3.1-py2.py3-none-any.whl", hash = "sha256:425e79b20939abbffa7633a91151a882aedc77564d9313e3584eb0416c28c558"},
+    {file = "pip-20.3.1.tar.gz", hash = "sha256:43f7d3811f05db95809d39515a5111dd05994965d870178a4fe10d5482f9d2e2"},
 ]
 "pip-shims 0.5.3" = [
     {file = "pip_shims-0.5.3-py2.py3-none-any.whl", hash = "sha256:16ca9f87485667b16b978b68a1aae4f9cc082c0fa018aed28567f9f34a590569"},
@@ -859,6 +865,10 @@ summary = "Backport of pathlib-compatible object wrapper for zip files"
     {file = "setuptools-50.3.2-py3-none-any.whl", hash = "sha256:2c242a0856fbad7efbe560df4a7add9324f340cf48df43651e9604924466794a"},
     {file = "setuptools-50.3.2.zip", hash = "sha256:ed0519d27a243843b05d82a5e9d01b0b083d9934eaa3d02779a23da18077bd3c"},
 ]
+"shellingham 1.3.2" = [
+    {file = "shellingham-1.3.2-py2.py3-none-any.whl", hash = "sha256:7f6206ae169dc1a03af8a138681b3f962ae61cc93ade84d0585cca3aaf770044"},
+    {file = "shellingham-1.3.2.tar.gz", hash = "sha256:576c1982bea0ba82fb46c36feb951319d7f42214a82634233f58b40d858a751e"},
+]
 "six 1.15.0" = [
     {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
     {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
@@ -941,5 +951,4 @@ summary = "Backport of pathlib-compatible object wrapper for zip files"
 
 [root]
 meta_version = "0.0.1"
-content_hash = "md5:320ab920bc7cee68a94fd078fda5541f"
-
+content_hash = "md5:cf7dd9fb5ea311d0fb65a7e99ba86e71"

pdm/cli/actions.py

 import json
 import os
 import shutil
+import sys
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Sequence
 
 import click
-import pythonfinder
-import tomlkit
 from pkg_resources import safe_name
 
 from pdm.cli.utils import (
@@ -28,6 +27,10 @@ from pdm.project import Project
 from pdm.resolver import resolve
 from pdm.utils import get_python_version
 
+PEP582_PATH = os.path.join(
+    os.path.dirname(sys.modules[__name__.split(".")[0]].__file__), "pep582"
+)
+
 
 def do_lock(
     project: Project,
@@ -97,7 +100,6 @@ def do_sync(
         candidates.update(project.get_locked_candidates())
     handler = project.core.synchronizer_class(candidates, project.environment)
     handler.synchronize(clean=clean, dry_run=dry_run)
-    project.environment.install_pep582_launcher()
 
 
 def do_add(
@@ -364,6 +366,8 @@ def do_init(
     python_requires: str = "",
 ) -> None:
     """Bootstrap the project and create a pyproject.toml"""
+    import tomlkit
+
     data = {
         "tool": {
             "pdm": {
@@ -388,13 +392,14 @@ def do_init(
         project._pyproject.setdefault("tool", {})["pdm"] = data["tool"]["pdm"]
         project._pyproject["build-system"] = data["build-system"]
     project.write_pyproject()
-    project.environment.install_pep582_launcher()
 
 
 def do_use(project: Project, python: str, first: bool = False) -> None:
     """Use the specified python version and save in project config.
     The python can be a version string or interpreter path.
     """
+    import pythonfinder
+
     if python and not all(c.isdigit() for c in python.split(".")):
         if Path(python).exists():
             python_path = Path(python).absolute().as_posix()
@@ -529,3 +534,29 @@ def ask_for_import(project: Project) -> None:
         return
     key, filepath = importable_files[int(choice)]
     do_import(project, filepath, key)
+
+
+def print_pep582_command(shell: str = "AUTO"):
+    """Print the export PYTHONPATH line to be evaluated by the shell."""
+    import shellingham
+
+    if shell == "AUTO":
+        shell = shellingham.detect_shell()[0]
+    shell = shell.lower()
+    lib_path = PEP582_PATH.replace("'", "\\'")
+    if shell in ("zsh", "bash"):
+        result = f"export PYTHONPATH='{lib_path}':$PYTHONPATH"
+    elif shell == "fish":
+        result = f"set -x PYTHONPATH '{lib_path}' $PYTHONPATH"
+    elif shell == "powershell":
+        result = f'$env:PYTHONPATH="{lib_path};$env:PYTHONPATH"'
+    elif shell == "cmd":
+        result = f"set PYTHONPATH={lib_path};%PYTHONPATH%"
+    elif shell in ("tcsh", "csh"):
+        result = f"setenv PYTHONPATH '{lib_path}':$PYTHONPATH"
+    else:
+        raise PdmUsageError(
+            f"Unsupported shell: {shell}, please specify another shell "
+            "via `--pep582 <SHELL>`"
+        )
+    stream.echo(result)

pdm/cli/commands/completion.py

@@ -19,7 +19,8 @@ class Command(BaseCommand):
         )
 
     def handle(self, project: Project, options: argparse.Namespace) -> None:
+        import shellingham
         from pycomplete import Completer
 
         completer = Completer(project.core.parser)
-        stream.echo(completer.render(options.shell))
+        stream.echo(completer.render(options.shell or shellingham.detect_shell()[0]))

pdm/cli/commands/run.py

@@ -7,6 +7,7 @@ import subprocess
 import sys
 from typing import Dict, List, Optional, Union
 
+from pdm.cli.actions import PEP582_PATH
 from pdm.cli.commands.base import BaseCommand
 from pdm.exceptions import PdmUsageError
 from pdm.iostream import stream
@@ -47,7 +48,11 @@ class Command(BaseCommand):
         env: Optional[Dict[str, str]] = None,
         env_file: Optional[str] = None,
     ) -> None:
-        os.environ.update({"PDM_PYTHON_PEP582": "1"})
+        if "PYTHONPATH" in os.environ:
+            new_path = os.sep.join([PEP582_PATH, os.getenv("PYTHONPATH")])
+        else:
+            new_path = PEP582_PATH
+        os.environ.update({"PYTHONPATH": new_path})
         if env_file:
             import dotenv
 

pdm/cli/options.py

@@ -64,7 +64,16 @@ dry_run_option = Option(
     "--dry-run",
     action="store_true",
     default=False,
-    help="Only prints actions without actually running them.",
+    help="Only prints actions without actually running them",
+)
+
+
+pep582_option = Option(
+    "--pep582",
+    const="AUTO",
+    metavar="SHELL",
+    nargs="?",
+    help="Print the command line to be eval'd by the shell",
 )
 
 sections_group = ArgumentGroup()

pdm/core.py

@@ -9,8 +9,9 @@ import click
 import pkg_resources
 from resolvelib import Resolver
 
+from pdm.cli.actions import print_pep582_command
 from pdm.cli.commands.base import BaseCommand
-from pdm.cli.options import verbose_option
+from pdm.cli.options import pep582_option, verbose_option
 from pdm.cli.utils import PdmFormatter, PdmParser
 from pdm.installers import Synchronizer
 from pdm.iostream import stream
@@ -57,6 +58,7 @@ class Core:
             help="show the version and exit",
         )
         verbose_option.add_to_parser(self.parser)
+        pep582_option.add_to_parser(self.parser)
 
         self.subparsers = self.parser.add_subparsers()
         for _, name, _ in pkgutil.iter_modules(COMMANDS_MODULE_PATH):
@@ -85,6 +87,9 @@ class Core:
             options.project = obj
         if options.global_project:
             options.project = options.global_project
+        if options.pep582:
+            print_pep582_command(options.pep582)
+            sys.exit(0)
         if not getattr(options, "project", None):
             options.project = self.project_class()
 

pdm/models/environment.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import collections
 import os
-import pkgutil
 import re
 import shutil
 import sys
@@ -390,23 +389,6 @@ class Environment:
                 re.sub(rb"#!.+?python.*?$", shebang, child.read_bytes(), flags=re.M)
             )
 
-    def install_pep582_launcher(self) -> None:
-        """Install a PEP 582 launcher to the site packages path
-        of given Python interperter.
-        """
-        lib_path = Path(get_sys_config_paths(self.python_executable)["purelib"])
-        if lib_path.joinpath("_pdm_pep582.pth").is_file():
-            stream.echo("PEP 582 launcher is ready.", verbosity=stream.DETAIL)
-            return
-        stream.echo("Installing PEP 582 launcher", verbosity=stream.DETAIL)
-        lib_path.joinpath("_pdm_pep582.py").write_bytes(
-            pkgutil.get_data(__name__, "../installers/_pep582.py")
-        )
-        lib_path.joinpath("_pdm_pep582.pth").write_text(
-            "import _pdm_pep582;_pdm_pep582.init()\n"
-        )
-        stream.echo("PEP 582 launcher is ready.", verbosity=stream.DETAIL)
-
 
 class GlobalEnvironment(Environment):
     """Global environment"""
@@ -427,6 +409,3 @@ class GlobalEnvironment(Environment):
     @property
     def packages_path(self) -> Optional[Path]:
         return None
-
-    def install_pep582_launcher(self):
-        pass

pdm/installers/_pep582.py → pdm/pep582/sitecustomize.py

@@ -2,10 +2,6 @@ import os
 import site
 import sys
 import warnings
-from distutils.sysconfig import get_python_lib
-
-# Global state to avoid recursive execution
-_initialized = False
 
 
 def get_pypackages_path(maxdepth=5):
@@ -36,54 +32,52 @@ def get_pypackages_path(maxdepth=5):
             return result
 
 
-def init():
-    global _initialized
-    if (
-        os.getenv("PDM_PYTHON_PEP582", "").lower() not in ("true", "1", "yes")
-        or _initialized
-    ):
-        # Do nothing if pep 582 is not enabled explicitly
-        return
-    _initialized = True
+def main():
+
+    self_path = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
+    sys.path[:] = [path for path in sys.path if os.path.normcase(path) != self_path]
+
     with_site_packages = os.getenv("PDM_WITH_SITE_PACKAGES")
+    needs_user_site = False
+    needs_site_packages = False
 
-    if sys.version_info[0] == 2 and getattr(sys, "argv", None) is None:
+    if getattr(sys, "argv", None) is None:
         warnings.warn(
             "PEP 582 can't be loaded based on the script path. "
             "As Python 2.7 reached the end of life on 2020/01/01, "
             "please upgrade to Python 3.",
         )
     else:
-        script_path = sys.argv[0]
-        if os.path.exists(script_path) and os.path.normcase(
-            os.path.abspath(script_path)
-        ).startswith(os.path.normcase(sys.prefix)):
-            with_site_packages = True
+        script_path = os.path.realpath(sys.argv[0])
+        needs_user_site = os.path.normcase(script_path).startswith(
+            os.path.normcase(site.USER_BASE)
+        )
+        needs_site_packages = any(
+            os.path.normcase(script_path).startswith(os.path.normcase(p))
+            for p in site.PREFIXES
+        )
+
     libpath = get_pypackages_path()
     if not libpath:
         return
-    if not with_site_packages:
-        # First, drop system-sites related paths.
-        original_sys_path = sys.path[:]
-        known_paths = set()
-        system_sites = {
-            os.path.normcase(site)
-            for site in (
-                get_python_lib(plat_specific=False),
-                get_python_lib(plat_specific=True),
-            )
-        }
-        for path in system_sites:
-            site.addsitedir(path, known_paths=known_paths)
-        system_paths = set(
-            os.path.normcase(path) for path in sys.path[len(original_sys_path) :]
-        )
-        original_sys_path = [
-            path
-            for path in original_sys_path
-            if os.path.normcase(path) not in system_paths
-        ]
-        sys.path = original_sys_path
+    # First, drop site related paths.
+    original_sys_path = sys.path[:]
+    paths_to_remove = set()
+    if not (with_site_packages or needs_user_site):
+        site.addusersitepackages(paths_to_remove)
+    if not (with_site_packages or needs_site_packages):
+        site.addsitepackages(paths_to_remove)
+    paths_to_remove = set(os.path.normcase(path) for path in paths_to_remove)
+    original_sys_path = [
+        path
+        for path in original_sys_path
+        if os.path.normcase(path) not in paths_to_remove
+    ]
+    sys.path[:] = original_sys_path
 
     # Second, add lib directories, ensuring .pth file are processed.
     site.addsitedir(libpath)
+
+
+main()
+del main

pdm/utils.py

@@ -29,7 +29,7 @@ from pip_shims.shims import (
 from pdm._types import Source
 
 if TYPE_CHECKING:
-    from pip_shims.compat import TCommand, TFinder, TSession, TShimmedFunc, Values
+    from pip_shims.compat import TCommand, TFinder, Values
 
 try:
     from functools import cached_property
@@ -100,17 +100,10 @@ def get_abi_tag(python_version):
 
 
 def get_package_finder(
-    install_cmd=None,  # type: Optional[TCommand]
+    install_cmd,  # type: TCommand
     options=None,  # type: Optional[Values]
-    session=None,  # type: Optional[TSession]
-    platform=None,  # type: Optional[str]
     python_version=None,  # type: Optional[Tuple[int, int]]
-    abi=None,  # type: Optional[str]
-    implementation=None,  # type: Optional[str]
-    target_python=None,  # type: Optional[Any]
     ignore_requires_python=None,  # type: Optional[bool]
-    target_python_builder=None,  # type: Optional[TShimmedFunc]
-    install_cmd_provider=None,  # type: Optional[TShimmedFunc]
 ):
     # type: (...) -> TFinder
     """Shim for compatibility to generate package finders.
@@ -118,85 +111,26 @@ def get_package_finder(
     Build and return a :class:`~pip._internal.index.package_finder.PackageFinder`
     instance using the :class:`~pip._internal.commands.install.InstallCommand` helper
     method to construct the finder, shimmed with backports as needed for compatibility.
-
-    :param install_cmd_provider: A shim for providing new install command instances.
-    :type install_cmd_provider: :class:`~pip_shims.models.ShimmedPathCollection`
-    :param install_cmd: A :class:`~pip._internal.commands.install.InstallCommand`
-        instance which is used to generate the finder.
-    :param optparse.Values options: An optional :class:`optparse.Values` instance
-        generated by calling `install_cmd.parser.parse_args()` typically.
-    :param session: An optional session instance, can be created by the `install_cmd`.
-    :param Optional[str] platform: An optional platform string, e.g. linux_x86_64
-    :param Optional[Tuple[str, ...]] python_version: A tuple of 2-digit strings
-        representing python versions, e.g. ("27", "35", "36", "37"...)
-    :param Optional[str] abi: The target abi to support, e.g. "cp38"
-    :param Optional[str] implementation: An optional implementation string for limiting
-        searches to a specific implementation, e.g. "cp" or "py"
-    :param target_python: A :class:`~pip._internal.models.target_python.TargetPython`
-        instance (will be translated to alternate arguments if necessary on incompatible
-        pip versions).
-    :param Optional[bool] ignore_requires_python: Whether to ignore `requires_python`
-        on resulting candidates, only valid after pip version 19.3.1
-    :param target_python_builder: A 'TargetPython' builder (e.g. the class itself,
-        uninstantiated)
-    :return: A :class:`pip._internal.index.package_finder.PackageFinder` instance
-    :rtype: :class:`pip._internal.index.package_finder.PackageFinder`
     """
-    from pip_shims.compat import get_session, resolve_possible_shim
+    from pip_shims.compat import get_session
 
-    if install_cmd is None:
-        install_cmd_provider = resolve_possible_shim(install_cmd_provider)
-        assert isinstance(install_cmd_provider, (type, functools.partial))
-        install_cmd = install_cmd_provider()
     if options is None:
         options, _ = install_cmd.parser.parse_args([])  # type: ignore
-    if session is None:
-        session = get_session(install_cmd=install_cmd, options=options)  # type: ignore
-    builder_args = inspect.getargs(
-        install_cmd._build_package_finder.__code__
-    )  # type: ignore
+    session = get_session(install_cmd=install_cmd, options=options)  # type: ignore
     build_kwargs = {"options": options, "session": session}
-    expects_targetpython = "target_python" in builder_args.args
-    received_python = any(
-        arg for arg in [platform, python_version, abi, implementation]
-    )
-    if expects_targetpython and received_python:
-        if not target_python:
-            if target_python_builder is None:
-                target_python_builder = TargetPython
-            if python_version and not abi:
-                abi = get_abi_tag(python_version)
-            target_python = target_python_builder(
-                platform=platform,
-                abi=abi,
-                implementation=implementation,
-                py_version_info=python_version,
-            )
+    if python_version:
+        target_python_builder = TargetPython
+        abi = get_abi_tag(python_version)
+        builder_args = inspect.signature(target_python_builder).parameters
+        target_python_params = {"py_version_info": python_version}
+        if "abi" in builder_args:
+            target_python_params["abi"] = abi
+        elif "abis" in builder_args:
+            target_python_params["abis"] = [abi]
+        target_python = target_python_builder(**target_python_params)
         build_kwargs["target_python"] = target_python
-    elif any(
-        arg in builder_args.args
-        for arg in ["platform", "python_version", "abi", "implementation"]
-    ):
-        if target_python and not received_python:
-            tags = target_python.get_tags()
-            version_impl = set([t[0] for t in tags])
-            # impls = set([v[:2] for v in version_impl])
-            # impls.remove("py")
-            # impl = next(iter(impls), "py") if not target_python
-            versions = set([v[2:] for v in version_impl])
-            build_kwargs.update(
-                {
-                    "platform": target_python.platform,
-                    "python_versions": versions,
-                    "abi": target_python.abi,
-                    "implementation": target_python.implementation,
-                }
-            )
-    if (
-        ignore_requires_python is not None
-        and "ignore_requires_python" in builder_args.args
-    ):
-        build_kwargs["ignore_requires_python"] = ignore_requires_python
+
+    build_kwargs["ignore_requires_python"] = ignore_requires_python
     return install_cmd._build_package_finder(**build_kwargs)  # type: ignore
 
 

pyproject.toml

@@ -39,6 +39,7 @@ importlib-metadata = {version = "*", marker = "python_version<'3.8'"}
 pep517 = "*"
 pycomplete = "<1.0.0,>=0.2.0"
 python-dotenv = "<1.0.0,>=0.15.0"
+shellingham = "<2.0.0,>=1.3.2"
 
 [tool.pdm.dev-dependencies]
 pytest = "*"

tests/cli/test_run.py

@@ -5,6 +5,7 @@ import textwrap
 
 import pytest
 
+from pdm.cli.actions import PEP582_PATH
 from pdm.utils import cd, temp_environ
 
 
@@ -29,7 +30,7 @@ def test_pep582_launcher_for_python_interpreter(project, invoke):
     )
     invoke(["add", "requests==2.24.0"], obj=project)
     env = os.environ.copy()
-    env.update({"PDM_PYTHON_PEP582": "1"})
+    env.update({"PYTHONPATH": PEP582_PATH})
     output = subprocess.check_output(
         [project.environment.python_executable, str(project.root.joinpath("main.py"))],
         env=env,

tests/conftest.py

@@ -232,6 +232,7 @@ def project_no_init(tmp_path, mocker):
     do_use(p, sys.executable)
     with temp_environ():
         os.environ.pop("VIRTUAL_ENV", None)
+        os.environ.pop("PYTHONPATH", None)
         yield p
     # Restore the config items
     Config._config_map = old_config_map