Merge pull request #222 from frostming/feature/219

Auto disable PEP 582 when a venv-like python is given

.github/workflows/ci.yml

@@ -53,10 +53,10 @@ jobs:
         with:
           path: __pypackages__
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-packages-${{ steps.set_variables.outputs.PY }}-${{ hashFiles('pdm.lock') }}
+          key: ${{ runner.os }}-${{ matrix.arch }}-packages-${{ steps.set_variables.outputs.PY }}-${{ hashFiles('pdm.lock') }}
           restore-keys: |
-            ${{ runner.os }}-packages-${{ steps.set_variables.outputs.PY }}-
-            ${{ runner.os }}-packages-
+            ${{ runner.os }}-${{ matrix.arch }}-packages-${{ steps.set_variables.outputs.PY }}-
+            ${{ runner.os }}-${{ matrix.arch }}-packages-
       - name: Install Current PDM
         run: |
           pip install -U pip

docs/docs/project.md

@@ -125,6 +125,8 @@ by the configuration item `use_venv`. When it is set to `True` PDM will use the
 - an virtualenv is already activated.
 - any of `venv`, `.venv`, `env` is an valid virtualenv folder.
 
+Besides, when `use-venv` is on and the interpreter path given is a venv-like path, PDM will reuse that venv directory as well.
+
 ## Import project metadata from existing project files
 
 If you are already other package manager tools like Pipenv or Poetry, it is easy to migrate to PDM.

news/219.feature

+Auto disable PEP 582 when a venv-like python is given as the interpreter path.

pdm/models/caches.py

 import hashlib
 import json
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Optional
-
-from pip._vendor import requests
+from typing import TYPE_CHECKING, Dict, Optional
 
 from pdm._types import CandidateInfo
 from pdm.exceptions import CorruptedCacheError
@@ -11,6 +9,8 @@ from pdm.models import pip_shims
 from pdm.utils import open_file
 
 if TYPE_CHECKING:
+    from pip._vendor import requests
+
     from pdm.models.candidates import Candidate
 
 
@@ -19,7 +19,7 @@ class CandidateInfoCache:
 
     def __init__(self, cache_file: Path) -> None:
         self.cache_file = cache_file
-        self._cache = {}  # type: Dict[str, Any]
+        self._cache = {}  # type: Dict[str, CandidateInfo]
         self._read_cache()
 
     def _read_cache(self) -> None:
@@ -80,7 +80,7 @@ class HashCache(pip_shims.SafeFileCache):
 
     Hashes are only cached when the URL appears to contain a hash in it and the
     cache key includes the hash value returned from the server). This ought to
-    avoid ssues where the location on the server changes.
+    avoid issues where the location on the server changes.
     """
 
     def __init__(self, *args, **kwargs):

pdm/models/environment.py

@@ -102,10 +102,8 @@ class Environment:
     def python_executable(self) -> str:
         """Get the Python interpreter path."""
         config = self.project.config
-        if config.get("python.path"):
-            return config["python.path"]
-        if PYENV_INSTALLED and config.get("python.use_pyenv", True):
-            return os.path.join(PYENV_ROOT, "shims", "python")
+        if self.project.project_config.get("python.path"):
+            return self.project.project_config["python.path"]
         if "VIRTUAL_ENV" in os.environ:
             stream.echo(
                 "An activated virtualenv is detected, reuse the interpreter now.",
@@ -113,6 +111,8 @@ class Environment:
                 verbosity=stream.DETAIL,
             )
             return get_venv_python(self.project.root)
+        if PYENV_INSTALLED and config.get("python.use_pyenv", True):
+            return os.path.join(PYENV_ROOT, "shims", "python")
 
         # First try what `python` refers to.
         path = shutil.which("python")
@@ -131,6 +131,9 @@ class Environment:
                 if self.python_requires.contains(version):
                     path = sys.executable
         if path:
+            if os.path.normcase(path) == os.path.normcase(sys.executable):
+                # Refer to the base interpreter to allow for venvs
+                path = getattr(sys, "_base_executable", sys.executable)
             stream.echo(
                 "Using Python interpreter: {} ({})".format(stream.green(path), version)
             )
@@ -414,7 +417,7 @@ class Environment:
         scripts = self.get_paths()["scripts"]
         maker = ScriptMaker(None, None)
         maker.executable = new_path
-        shebang = maker._get_shebang("utf-8").rstrip()
+        shebang = maker._get_shebang("utf-8").rstrip().replace(b"\\", b"\\\\")
         for child in Path(scripts).iterdir():
             if not child.is_file() or child.suffix not in (".exe", ".py", ""):
                 continue

pdm/project/config.py

@@ -149,15 +149,16 @@ class Config(MutableMapping):
             fp.write(tomlkit.dumps(toml_data))
 
     def __getitem__(self, key: str) -> Any:
-        if key not in self._data:
-            raise NoConfigError(key)
         env_var = self._config_map[key].env_var
         if env_var is not None and env_var in os.environ:
             env_value = os.environ[env_var]
             if isinstance(self._config_map[key].default, bool):
                 env_value = ensure_boolean(env_value)
             return env_value
-        return self._data[key]
+        try:
+            return self._data[key]
+        except KeyError:
+            raise NoConfigError(key) from None
 
     def __setitem__(self, key: str, value: Any) -> None:
         if key not in self._config_map:
@@ -176,7 +177,7 @@ class Config(MutableMapping):
             stream.echo(
                 stream.yellow(
                     "WARNING: the config is shadowed by env var '{}', "
-                    "set value won't take effect.".format(env_var)
+                    "the value set won't take effect.".format(env_var)
                 )
             )
         self._data[key] = value

pdm/project/core.py

@@ -142,11 +142,19 @@ class Project:
                 "==" + get_python_version(env.python_executable, True)[0]
             )
             return env
-        if self.config["use_venv"]:
+        if not self.project_config.get("python.path") and self.config["use_venv"]:
             venv_python = get_venv_python(self.root)
             if venv_python:
                 self.project_config["python.path"] = venv_python
-                return GlobalEnvironment(self)
+        if (
+            self.config["use_venv"]
+            and self.project_config.get("python.path")
+            and Path(self.project_config.get("python.path"))
+            .parent.parent.joinpath("pyvenv.cfg")
+            .exists()
+        ):
+            # Only recognize venv created by python -m venv and virtualenv>20
+            return GlobalEnvironment(self)
         return Environment(self)
 
     @property

pdm/utils.py

@@ -10,7 +10,7 @@ import tempfile
 import urllib.parse as parse
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Generic, List, Optional, Tuple, TypeVar, Union
 
 from distlib.wheel import Wheel
 from tomlkit.container import Container
@@ -29,13 +29,15 @@ try:
     from functools import cached_property
 except ImportError:
 
-    class cached_property:
-        def __init__(self, func):
+    _T = TypeVar("_T")
+
+    class cached_property(Generic[_T]):
+        def __init__(self, func: Callable[[Any], _T]):
             self.func = func
             self.attr_name = func.__name__
             self.__doc__ = func.__doc__
 
-        def __get__(self, inst, cls=None):
+        def __get__(self, inst: Any, cls=None) -> _T:
             if inst is None:
                 return self
             if self.attr_name not in inst.__dict__:

tests/conftest.py

@@ -229,7 +229,7 @@ def project_no_init(tmp_path, mocker):
     mocker.patch("pdm.project.core.Config.HOME_CONFIG", tmp_path)
     old_config_map = Config._config_map.copy()
     p.global_config["cache_dir"] = tmp_path.joinpath("caches").as_posix()
-    do_use(p, sys.executable)
+    do_use(p, getattr(sys, "_base_executable", sys.executable))
     with temp_environ():
         os.environ.pop("VIRTUAL_ENV", None)
         os.environ.pop("PYTHONPATH", None)

tests/test_project.py

 import os
 import sys
+import venv
 from pathlib import Path
 
 import distlib.wheel
@@ -57,19 +58,17 @@ def test_global_project(tmp_path):
     assert project.environment.is_global
 
 
-def test_project_use_venv(project, mocker):
+def test_project_use_venv(project):
     del project.project_config["python.path"]
     scripts = "Scripts" if os.name == "nt" else "bin"
     suffix = ".exe" if os.name == "nt" else ""
-
-    os.environ["VIRTUAL_ENV"] = "/path/to/env"
-    mocker.patch("pdm.models.environment.get_python_version", return_value="3.7.0")
+    venv.create(project.root / "venv")
 
     project.project_config["use_venv"] = True
     env = project.environment
     assert (
         Path(env.python_executable)
-        == Path("/path/to/env") / scripts / f"python{suffix}"
+        == project.root / "venv" / scripts / f"python{suffix}"
     )
     assert env.is_global
 
@@ -95,3 +94,18 @@ def test_project_packages_path(project):
         assert packages_path.name == version + "-32"
     else:
         assert packages_path.name == version
+
+
+def test_project_auto_detect_venv(project):
+
+    venv.create(project.root / "test_venv")
+
+    scripts = "Scripts" if os.name == "nt" else "bin"
+    suffix = ".exe" if os.name == "nt" else ""
+
+    project.project_config["use_venv"] = True
+    project.project_config["python.path"] = (
+        project.root / "test_venv" / scripts / f"python{suffix}"
+    ).as_posix()
+
+    assert project.environment.is_global