New method of enabling PEP 582

pdm/cli/actions.py

@@ -97,6 +97,7 @@ def do_sync(
         candidates.update(project.get_locked_candidates())
     handler = project.core.synchronizer_class(candidates, project.environment)
     handler.synchronize(clean=clean, dry_run=dry_run)
+    project.environment.install_pep582_launcher()
 
 
 def do_add(
@@ -387,7 +388,7 @@ def do_init(
         project._pyproject.setdefault("tool", {})["pdm"] = data["tool"]["pdm"]
         project._pyproject["build-system"] = data["build-system"]
     project.write_pyproject()
-    project.environment.write_site_py()
+    project.environment.install_pep582_launcher()
 
 
 def do_use(project: Project, python: str, first: bool = False) -> None:

pdm/cli/commands/run.py

@@ -11,7 +11,6 @@ from pdm.cli.commands.base import BaseCommand
 from pdm.exceptions import PdmUsageError
 from pdm.iostream import stream
 from pdm.project import Project
-from pdm.utils import find_project_root
 
 
 class Command(BaseCommand):
@@ -42,37 +41,35 @@ class Command(BaseCommand):
         env: Optional[Dict[str, str]] = None,
         env_file: Optional[str] = None,
     ) -> None:
-        with project.environment.activate():
-            if env_file:
-                import dotenv
+        os.environ.update({"PYTHONPEP582": "1"})
+        if env_file:
+            import dotenv
 
-                stream.echo(f"Loading .env file: {stream.green(env_file)}", err=True)
-                dotenv.load_dotenv(
-                    project.root.joinpath(env_file).as_posix(), override=True
-                )
-            if env:
-                os.environ.update(env)
-            if shell:
-                sys.exit(subprocess.call(os.path.expandvars(args), shell=True))
+            stream.echo(f"Loading .env file: {stream.green(env_file)}", err=True)
+            dotenv.load_dotenv(
+                project.root.joinpath(env_file).as_posix(), override=True
+            )
+        if env:
+            os.environ.update(env)
+        if shell:
+            sys.exit(subprocess.call(os.path.expandvars(args), shell=True))
 
-            command, *args = args
-            expanded_command = project.environment.which(command)
-            if not expanded_command:
-                raise PdmUsageError(
-                    "Command {} is not found on your PATH.".format(
-                        stream.green(f"'{command}'")
-                    )
+        command, *args = args
+        expanded_command = project.environment.which(command)
+        if not expanded_command:
+            raise PdmUsageError(
+                "Command {} is not found on your PATH.".format(
+                    stream.green(f"'{command}'")
                 )
-            expanded_command = os.path.expanduser(os.path.expandvars(expanded_command))
-            expanded_args = [
-                os.path.expandvars(arg) for arg in [expanded_command] + args
-            ]
-            if os.name == "nt" or "CI" in os.environ:
-                # In order to make sure pytest is playing well,
-                # don't hand over the process under a testing environment.
-                sys.exit(subprocess.call(expanded_args))
-            else:
-                os.execv(expanded_command, expanded_args)
+            )
+        expanded_command = os.path.expanduser(os.path.expandvars(expanded_command))
+        expanded_args = [os.path.expandvars(arg) for arg in [expanded_command] + args]
+        if os.name == "nt" or "CI" in os.environ:
+            # In order to make sure pytest is playing well,
+            # don't hand over the process under a testing environment.
+            sys.exit(subprocess.call(expanded_args))
+        else:
+            os.execv(expanded_command, expanded_args)
 
     def _normalize_script(self, script):
         if not getattr(script, "items", None):
@@ -153,17 +150,6 @@ class Command(BaseCommand):
         global_env_options = project.scripts.get("_", {}) if project.scripts else {}
         if project.scripts and options.command in project.scripts:
             self._run_script(project, options.command, options.args, global_env_options)
-        elif os.path.isfile(options.command) and options.command.endswith(".py"):
-            # Allow executing py scripts like `pdm run my_script.py`.
-            # In this case, the nearest `__pypackages__` will be loaded as
-            # the library source.
-            new_root = find_project_root(os.path.abspath(options.command))
-            project = Project(new_root) if new_root else project
-            self._run_command(
-                project,
-                ["python", options.command] + options.args,
-                **global_env_options,
-            )
         else:
             self._run_command(
                 project, [options.command] + options.args, **global_env_options

pdm/installers/_editable_install.py

@@ -3,14 +3,6 @@ import os
 import sys
 import tokenize
 
-from setuptools.command import easy_install
-
-EXE_INITIALIZE = """
-import sys
-with open({0!r}) as fp:
-    exec(compile(fp.read(), __file__, "exec"))
-""".strip()
-
 
 def install(setup_py, prefix, lib_dir, bin_dir):
     __file__ = setup_py
@@ -30,16 +22,6 @@ def install(setup_py, prefix, lib_dir, bin_dir):
         "--site-dirs={0}".format(lib_dir),
     ]
     sys.path.append(lib_dir)
-    if os.getenv("INJECT_SITE", "").lower() in ("1", "true", "yes"):
-        # Patches the script writer to inject library path
-        easy_install.ScriptWriter.template = easy_install.ScriptWriter.template.replace(
-            "import sys",
-            EXE_INITIALIZE.format(
-                os.path.abspath(
-                    os.path.join(lib_dir, os.path.pardir, "site/sitecustomize.py")
-                )
-            ),
-        )
     exec(compile(code, __file__, "exec"))
 
 

pdm/installers/_pep582.py

+import os
+import site
+import sys
+from distutils.sysconfig import get_python_lib
+
+# Global state to avoid recursive execution
+_initialized = False
+
+
+def get_pypackages_path(maxdepth=5):
+    def find_pypackage(path):
+        packages_name = "__pypackages__/{}/lib".format(
+            ".".join(map(str, sys.version_info[:2]))
+        )
+        for _ in range(maxdepth):
+            if os.path.exists(os.path.join(path, packages_name)):
+                return os.path.join(path, packages_name)
+            if os.path.dirname(path) == path:
+                # Root path is reached
+                break
+            path = os.path.dirname(path)
+        return None
+
+    script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
+    return find_pypackage(script_dir) or find_pypackage(os.getcwd())
+
+
+def init():
+    global _initialized
+    if (
+        os.getenv("PYTHONPEP582", "").lower() not in ("true", "1", "yes")
+        or _initialized
+    ):
+        # Do nothing if pep 582 is not enabled explicitly
+        return
+    _initialized = True
+    # First, drop system-sites related paths.
+    libpath = get_pypackages_path()
+    if not libpath:
+        return
+    original_sys_path = sys.path[:]
+    known_paths = set()
+    system_sites = {
+        os.path.normcase(site)
+        for site in (
+            get_python_lib(plat_specific=False),
+            get_python_lib(plat_specific=True),
+        )
+    }
+    for path in system_sites:
+        site.addsitedir(path, known_paths=known_paths)
+    system_paths = set(
+        os.path.normcase(path) for path in sys.path[len(original_sys_path) :]
+    )
+    original_sys_path = [
+        path for path in original_sys_path if os.path.normcase(path) not in system_paths
+    ]
+    sys.path = original_sys_path
+
+    # Second, add lib directories, ensuring .pth file are processed.
+    site.addsitedir(libpath)

pdm/installers/installers.py

@@ -32,13 +32,6 @@ def format_dist(dist: Distribution) -> str:
     return formatter.format(version=stream.yellow(dist.version), path=path)
 
 
-EXE_INITIALIZE = """
-import sys
-with open({0!r}) as fp:
-    exec(compile(fp.read(), __file__, "exec"))
-""".strip()
-
-
 class Installer:  # pragma: no cover
     """The installer that performs the installation and uninstallation actions."""
 
@@ -59,14 +52,6 @@ class Installer:  # pragma: no cover
         paths = self.environment.get_paths()
         maker = distlib.scripts.ScriptMaker(None, None)
         maker.executable = self.environment.python_executable
-        if not self.environment.is_global:
-            site_custom_script = (
-                self.environment.packages_path / "site/sitecustomize.py"
-            ).as_posix()
-            maker.script_template = maker.script_template.replace(
-                "import sys",
-                EXE_INITIALIZE.format(site_custom_script),
-            )
         wheel.install(paths, maker)
 
     def install_editable(self, ireq: shims.InstallRequirement) -> None:

pdm/installers/synchronizers.py

@@ -234,8 +234,6 @@ class Synchronizer:
         if not clean:
             to_remove = []
         if not any([to_add, to_update, to_remove]):
-            if not dry_run:
-                self.environment.write_site_py()
             stream.echo(
                 stream.yellow("All packages are synced to date, nothing to do.")
             )
@@ -302,7 +300,7 @@ class Synchronizer:
                 stream.echo(stream.red("\nERRORS:"))
                 stream.echo("".join(errors), err=True)
                 raise InstallationError("Some package operations are not complete yet")
-            self.environment.write_site_py()
+
             if install_self:
                 stream.echo("Installing the project as an editable package...")
                 with stream.indent("  "):

pdm/models/builders.py

@@ -8,6 +8,7 @@ import subprocess
 import sys
 import tempfile
 import threading
+from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Iterable, List, Optional
 
@@ -136,10 +137,9 @@ class EnvBuilder:
     def __init__(self, src_dir: os.PathLike, environment: Environment) -> None:
         self._env = environment
         self._path = None  # type: Optional[str]
+        self._saved_env = None
         self.executable = self._env.python_executable
-        self.pip_command = self._get_pip_command()
         self.src_dir = src_dir
-        self._saved_env = None
 
         try:
             with open(os.path.join(src_dir, "pyproject.toml")) as f:
@@ -170,6 +170,10 @@ class EnvBuilder:
             python_executable=self.executable,
         )
 
+    @cached_property
+    def pip_command(self):
+        return self._get_pip_command()
+
     def subprocess_runner(self, cmd, cwd=None, extra_environ=None):
         env = self._saved_env.copy() if self._saved_env else {}
         if extra_environ:
@@ -215,6 +219,7 @@ class EnvBuilder:
             if not old_path
             else os.pathsep.join([paths["scripts"], old_path]),
             "PYTHONNOUSERSITE": "1",
+            "PYTHONPEP582": "0",
         }
         stream.logger.debug("Preparing isolated env for PEP 517 build...")
         return self

pdm/models/environment.py

@@ -2,11 +2,11 @@ from __future__ import annotations
 
 import collections
 import os
+import pkgutil
 import re
 import shutil
 import sys
 import sysconfig
-import textwrap
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
@@ -154,7 +154,7 @@ class Environment:
         return paths
 
     @contextmanager
-    def activate(self, site_packages: bool = False):
+    def activate(self):
         """Activate the environment. Manipulate the ``PYTHONPATH`` and patches ``pip``
         to be aware of local packages. This method acts like a context manager.
 
@@ -162,21 +162,6 @@ class Environment:
         """
         paths = self.get_paths()
         with temp_environ():
-            python_root = os.path.dirname(self.python_executable)
-            os.environ.update(
-                {
-                    "PATH": os.pathsep.join(
-                        [python_root, paths["scripts"], os.environ["PATH"]]
-                    ),
-                    "PYTHONNOUSERSITE": "1",
-                }
-            )
-            if self.packages_path:
-                os.environ.update(
-                    {"PYTHONPATH": (self.packages_path / "site").as_posix()}
-                )
-            if site_packages:
-                os.environ["PDM_SITE_PACKAGES"] = "1"
             working_set = self.get_working_set()
             _old_ws = pkg_resources.working_set
             pkg_resources.working_set = working_set.pkg_ws
@@ -216,7 +201,7 @@ class Environment:
             / ".".join(map(str, get_python_version(self.python_executable)[:2]))
         )
         scripts = "Scripts" if os.name == "nt" else "bin"
-        for subdir in [scripts, "include", "lib", "site"]:
+        for subdir in [scripts, "include", "lib"]:
             pypackages.joinpath(subdir).mkdir(exist_ok=True, parents=True)
         return pypackages
 
@@ -387,7 +372,10 @@ class Environment:
             if not version or this_version.startswith(version):
                 return self.python_executable
         # Fallback to use shutil.which to find the executable
-        return shutil.which(command, path=os.getenv("PATH"))
+        this_path = self.get_paths()["scripts"]
+        python_root = os.path.dirname(self.python_executable)
+        new_path = os.pathsep.join([python_root, this_path, os.getenv("PATH", "")])
+        return shutil.which(command, path=new_path)
 
     def update_shebangs(self, new_path: str) -> None:
         """Update the shebang lines"""
@@ -402,45 +390,22 @@ class Environment:
                 re.sub(rb"#!.+?python.*?$", shebang, child.read_bytes(), flags=re.M)
             )
 
-    def write_site_py(self) -> None:
-        """Write a custom site.py into the package library folder."""
-        lib_dir = self.get_paths()["purelib"]
-        dest_path = self.packages_path / "site/sitecustomize.py"
-
-        template = textwrap.dedent(
-            """
-            import os, site, sys
-            from distutils.sysconfig import get_python_lib
-
-            # First, drop system-sites related paths.
-            original_sys_path = sys.path[:]
-            known_paths = set()
-            system_sites = {{
-                os.path.normcase(site) for site in (
-                    get_python_lib(plat_specific=False),
-                    get_python_lib(plat_specific=True),
-                )
-            }}
-            for path in system_sites:
-                site.addsitedir(path, known_paths=known_paths)
-            system_paths = set(
-                os.path.normcase(path)
-                for path in sys.path[len(original_sys_path):]
-            )
-            if "PDM_SITE_PACKAGES" not in os.environ:
-                original_sys_path = [
-                    path for path in original_sys_path
-                    if os.path.normcase(path) not in system_paths
-                ]
-                sys.path = original_sys_path
-
-            # Second, add lib directories.
-            # ensuring .pth file are processed.
-            for path in {lib_dirs!r}:
-                site.addsitedir(path)
-            """
+    def install_pep582_launcher(self) -> None:
+        """Install a PEP 582 launcher to the site packages path
+        of given Python interperter.
+        """
+        lib_path = Path(get_sys_config_paths(self.python_executable)["purelib"])
+        if lib_path.joinpath("_pdm_pep582.pth").is_file():
+            stream.echo("PEP 582 launcher is ready.", verbosity=stream.DETAIL)
+            return
+        stream.echo("Installing PEP 582 launcher", verbosity=stream.DETAIL)
+        lib_path.joinpath("_pdm_pep582.py").write_bytes(
+            pkgutil.get_data(__name__, "../installers/_pep582.py")
+        )
+        lib_path.joinpath("_pdm_pep582.pth").write_text(
+            "import _pdm_pep582;_pdm_pep582.init()\n"
         )
-        dest_path.write_text(template.format(lib_dirs=[lib_dir]))
+        stream.echo("PEP 582 launcher is ready.", verbosity=stream.DETAIL)
 
 
 class GlobalEnvironment(Environment):
@@ -463,5 +428,5 @@ class GlobalEnvironment(Environment):
     def packages_path(self) -> Optional[Path]:
         return None
 
-    def write_site_py(self) -> None:
-        return None
+    def install_pep582_launcher(self):
+        pass

setup.cfg

@@ -9,6 +9,9 @@ exclude =
     __pypackages__,
     temp_script.py
 max_line_length = 88
+ignore =
+    E203
+    W503
 
 [coverage:run]
 branch = true

tests/cli/test_run.py

 import json
 import os
+import subprocess
 import textwrap
 
 import pytest
@@ -8,15 +9,34 @@ from pdm.utils import cd, temp_environ
 
 
 def test_pep582_not_loading_site_packages(project, invoke, capfd):
-    invoke(
-        ["run", "python", "-c", "import sys,json;print(json.dumps(sys.path))"],
-        obj=project,
-    )
+    with cd(project.root):
+        invoke(["install"], obj=project)
+        invoke(
+            ["run", "python", "-c", "import sys,json;print(json.dumps(sys.path))"],
+            obj=project,
+        )
     sys_path = json.loads(capfd.readouterr()[0])
     assert not any("site-packages" in p for p in sys_path)
     assert str(project.environment.packages_path / "lib") in sys_path
 
 
+@pytest.mark.pypi
+def test_pep582_launcher_for_python_interpreter(project, invoke):
+    project.tool_settings["python_requires"] = ">=3.6"
+    project.write_pyproject()
+    project.root.joinpath("main.py").write_text(
+        "import requests\nprint(requests.__version__)\n"
+    )
+    invoke(["add", "requests==2.24.0"], obj=project)
+    env = os.environ.copy()
+    env.update({"PYTHONPEP582": "1"})
+    output = subprocess.check_output(
+        [project.environment.python_executable, str(project.root.joinpath("main.py"))],
+        env=env,
+    )
+    assert output.decode().strip() == "2.24.0"
+
+
 def test_run_command_not_found(invoke):
     result = invoke(["run", "foobar"])
     assert "Command 'foobar' is not found on your PATH." in result.stderr
@@ -172,21 +192,3 @@ def test_run_show_list_of_scripts(project, invoke):
         == "test_script call  test_script:main call a python function"
     )
     assert result_lines[2].strip() == "test_shell  shell echo $FOO        shell command"
-
-
-@pytest.mark.pypi
-def test_run_script_with_pep582(project, invoke, capfd):
-    project.tool_settings["python_requires"] = ">=3.7"
-    project.write_pyproject()
-    (project.root / "test_script.py").write_text(
-        "import requests\nprint(requests.__version__)\n"
-    )
-    result = invoke(["add", "requests==2.24.0"], obj=project)
-    assert result.exit_code == 0
-    capfd.readouterr()
-
-    with cd(os.path.expanduser("~")):
-        result = invoke(["run", str(project.root / "test_script.py")], obj=project)
-        assert result.exit_code == 0
-        out, _ = capfd.readouterr()
-        assert out.strip() == "2.24.0"