Fix editable install and uninstall

pdm.lock

@@ -13,7 +13,7 @@ summary = "Atomic file writes."
 
 [[package]]
 name = "attrs"
-sections = ["default", "dev"]
+sections = ["dev", "default"]
 version = "19.3.0"
 marker = "python_version >= \"2.7\""
 summary = "Classes Without Boilerplate"
@@ -34,7 +34,7 @@ summary = "Composable command line interface toolkit"
 
 [[package]]
 name = "colorama"
-sections = ["default", "dev"]
+sections = ["dev", "default"]
 version = "0.4.3"
 marker = "python_version >= \"2.7\""
 summary = "Cross-platform colored terminal text."
@@ -43,13 +43,14 @@ summary = "Cross-platform colored terminal text."
 name = "coverage"
 sections = ["dev"]
 version = "4.4.2"
-marker = "python_version >= \"2.7\" and python_version < \"4.0\" and python_version not in \"3.0, 3.1, 3.2, 3.3, 3.4\""
+marker = "python_version < \"4.0\" and python_version >= \"2.7\" and python_version not in \"3.0, 3.1, 3.2, 3.3, 3.4\""
 summary = "Code coverage measurement for Python"
 
 [[package]]
 name = "crayons"
 sections = ["default"]
 version = "0.3.0"
+marker = "python_version >= \"2.7\" and python_version not in \"3.0, 3.1, 3.2, 3.3\""
 summary = "TextUI colors for Python."
 
 [package.dependencies]
@@ -85,7 +86,7 @@ six = ">=1.12.0"
 name = "importlib-metadata"
 sections = ["dev"]
 version = "1.4.0"
-marker = "python_version < \"3.8\" and python_version >= \"3.5\""
+marker = "python_version >= \"3.5\" and python_version < \"3.8\""
 summary = "Read metadata from Python packages"
 
 [package.dependencies]
@@ -109,7 +110,7 @@ summary = "More routines for operating on iterables, beyond itertools"
 
 [[package]]
 name = "packaging"
-sections = ["default", "dev"]
+sections = ["dev", "default"]
 version = "20.0"
 marker = "python_version >= \"2.7\""
 summary = "Core utilities for Python packages"
@@ -159,7 +160,7 @@ summary = "library with cross-python path, ini-parsing, io, code, log facilities
 
 [[package]]
 name = "pyparsing"
-sections = ["default", "dev"]
+sections = ["dev", "default"]
 version = "2.4.6"
 marker = "python_version >= \"2.7\" and python_version not in \"3.0, 3.1, 3.2\""
 summary = "Python parsing module"
@@ -228,7 +229,7 @@ summary = "Easily download, build, install, upgrade, and uninstall Python packag
 
 [[package]]
 name = "six"
-sections = ["default", "dev"]
+sections = ["dev", "default"]
 version = "1.14.0"
 marker = "python_version >= \"2.7\" and python_version not in \"3.0, 3.1, 3.2\""
 summary = "Python 2 and 3 compatibility utilities"
@@ -302,7 +303,7 @@ summary = "Yet Another Terminal Spinner"
 name = "zipp"
 sections = ["dev"]
 version = "2.0.0"
-marker = "python_version < \"3.8\" and python_version >= \"3.6\""
+marker = "python_version >= \"3.6\" and python_version < \"3.8\""
 summary = "Backport of pathlib-compatible object wrapper for zip files"
 
 [package.dependencies]
@@ -477,5 +478,5 @@ more-itertools = "*"
 ]
 
 [root]
-content_hash = "md5:4b45d0f13be19f09d14bf4afb278ab9f"
+content_hash = "md5:9ae5428069db95310550281f82109e1d"
 meta_version = "0.0.1"

pdm/__init__.py

-__version__ = "0.0.3"
+__version__ = "0.0.4"

pdm/_editable_install.py

+from setuptools.command import easy_install
+import sys
+import os
+import tokenize
+
+
+def install(setup_py, prefix):
+    __file__ = setup_py
+    bin_dir = "Scripts" if os.name == "nt" else "bin"
+    install_dir = os.path.join(prefix, "lib")
+    scripts_dir = os.path.join(prefix, bin_dir)
+
+    with getattr(tokenize, "open", open)(setup_py) as f:
+        code = f.read().replace("\\r\\n", "\n")
+    sys.argv[1:] = [
+        "develop",
+        f"--install-dir={install_dir}",
+        "--no-deps",
+        f"--prefix={prefix}",
+        f"--script-dir={scripts_dir}",
+        f"--site-dirs={install_dir}",
+    ]
+    # Patches the script writer to inject library path
+    easy_install.ScriptWriter.template = easy_install.ScriptWriter.template.replace(
+        "import sys",
+        "import sys\nsys.path.insert(0, {!r})".format(install_dir.replace("\\", "/")),
+    )
+    exec(compile(code, __file__, "exec"))
+
+
+if __name__ == "__main__":
+    setup_py, prefix = sys.argv[1:3]
+    install(setup_py, prefix)

pdm/cli/actions.py

@@ -287,8 +287,7 @@ def do_remove(
 
 def do_list(project: Project) -> None:
     working_set = project.environment.get_working_set()
-    for key in working_set:
-        dist = working_set[key][0]
+    for key, dist in working_set.items():
         context.io.echo(format_dist(dist))
 
 

pdm/installers.py

+import importlib
 import subprocess
 from typing import Dict, List, Tuple
 
@@ -11,7 +12,7 @@ from vistir import cd
 from pdm.context import context
 from pdm.models.candidates import Candidate
 from pdm.models.environment import Environment
-from pdm.models.requirements import strip_extras
+from pdm.models.requirements import strip_extras, parse_requirement
 
 SETUPTOOLS_SHIM = (
     "import setuptools, tokenize;__file__=%r;"
@@ -69,34 +70,38 @@ class Installer:
         scripts.executable = self.environment.python_executable
         scripts.script_template = scripts.script_template.replace(
             "import sys",
-            "import sys; sys.path.insert(0, {!r})".format(paths["platlib"]),
+            "import sys\nsys.path.insert(0, {!r})".format(paths["platlib"]),
         )
         wheel.install(paths, scripts)
 
     def install_editable(self, ireq: shims.InstallRequirement) -> None:
         setup_path = ireq.setup_py_path
         paths = self.environment.get_paths()
+        install_script = importlib.import_module(
+            "pdm._editable_install"
+        ).__file__.strip("co")
         install_args = [
             self.environment.python_executable,
             "-u",
-            "-c",
-            SETUPTOOLS_SHIM % setup_path,
-            "develop",
-            "--install-dir={}".format(paths["platlib"]),
-            "--no-deps",
-            "--prefix={}".format(paths["prefix"]),
-            "--script-dir={}".format(paths["scripts"]),
-            "--site-dirs={}".format(paths["platlib"]),
+            install_script,
+            setup_path,
+            paths["prefix"],
         ]
         with self.environment.activate(), cd(ireq.unpacked_source_directory):
             subprocess.check_call(install_args)
 
     def uninstall(self, name: str) -> None:
         working_set = self.environment.get_working_set()
-        ireq = shims.install_req_from_line(name)
+        dist = working_set[name]
+        req = parse_requirement(name)
+        if _is_dist_editable(dist):
+            ireq = shims.install_req_from_editable(dist.location)
+        else:
+            ireq = shims.install_req_from_line(name)
+        ireq.req = req
         context.io.echo(
             f"Uninstalling: {context.io.green(name, bold=True)} "
-            f"{context.io.yellow(working_set.by_key[name].version)}"
+            f"{context.io.yellow(working_set[name].version)}"
         )
 
         with self.environment.activate():
@@ -123,12 +128,11 @@ class Synchronizer:
         to_update, to_remove = [], []
         candidates = self.candidates.copy()
         environment = self.environment.marker_environment()
-        for key in working_set:
+        for key, dist in working_set.items():
             if key not in candidates:
                 to_remove.append(key)
             else:
                 can = candidates.pop(key)
-                dist = working_set[key][0]
                 if can.marker and not can.marker.evaluate(environment):
                     to_remove.append(key)
                 elif not _is_dist_editable(dist) and dist.version != can.version:
@@ -139,7 +143,7 @@ class Synchronizer:
                 strip_extras(name)[0]
                 for name, can in candidates.items()
                 if not (can.marker and not can.marker.evaluate(environment))
-                and not working_set[strip_extras(name)[0]]
+                and strip_extras(name)[0] not in working_set
             }
         )
         return to_add, to_update, to_remove

pdm/models/environment.py

 from __future__ import annotations
 
+import collections
 import os
 import sys
 import sysconfig
 from contextlib import contextmanager
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Iterator
 
 from pip._internal.req import req_uninstall
 from pip._internal.utils import misc
 from pip._vendor import pkg_resources
 from pip_shims import shims
+from pythonfinder import Finder
+from vistir.contextmanagers import temp_environ
+from vistir.path import normalize_path
 
 from pdm.context import context
 from pdm.exceptions import NoPythonVersion
@@ -23,9 +27,6 @@ from pdm.utils import (
     get_python_version,
     get_pep508_environment,
 )
-from pythonfinder import Finder
-from vistir.contextmanagers import temp_environ
-from vistir.path import normalize_path
 
 if TYPE_CHECKING:
     from pdm.models.specifiers import PySpecSet
@@ -33,6 +34,41 @@ if TYPE_CHECKING:
     from pdm._types import Source
 
 
+class WorkingSet(collections.abc.Mapping):
+    """A dict-like class that holds all installed packages in the lib directory."""
+
+    def __init__(
+        self,
+        paths: Optional[List[str]] = None,
+        python: Tuple[int, ...] = sys.version_info[:3],
+    ):
+        self.env = pkg_resources.Environment(paths, python=python)
+        self.pkg_ws = pkg_resources.WorkingSet(paths)
+        self.__add_editable_dists()
+
+    def __getitem__(self, key: str) -> pkg_resources.Distribution:
+        rv = self.env[key]
+        if rv:
+            return rv[0]
+        else:
+            raise KeyError(key)
+
+    def __len__(self) -> int:
+        return len(self.env._distmap)
+
+    def __iter__(self) -> Iterator[str]:
+        for item in self.env:
+            yield item
+
+    def __add_editable_dists(self):
+        """Editable distributions are not present in pkg_resources.WorkingSet,
+        Get them from self.env
+        """
+        missing_keys = [key for key in self if key not in self.pkg_ws.by_key]
+        for key in missing_keys:
+            self.pkg_ws.add(self[key])
+
+
 class Environment:
     def __init__(self, python_requires: PySpecSet, config: Config) -> None:
         self.python_requires = python_requires
@@ -78,9 +114,9 @@ class Environment:
         with temp_environ():
             old_paths = os.getenv("PYTHONPATH")
             if old_paths:
-                new_paths = os.pathsep.join([paths["platlib"], old_paths])
+                new_paths = os.pathsep.join([paths["purelib"], old_paths])
             else:
-                new_paths = paths["platlib"]
+                new_paths = paths["purelib"]
             os.environ["PYTHONPATH"] = new_paths
             python_root = os.path.dirname(self.python_executable)
             os.environ["PATH"] = os.pathsep.join(
@@ -88,14 +124,17 @@ class Environment:
             )
             working_set = self.get_working_set()
             _old_ws = pkg_resources.working_set
-            pkg_resources.working_set = working_set
+            pkg_resources.working_set = working_set.pkg_ws
             # HACK: Replace the is_local with environment version so that packages can
             # be removed correctly.
+            _old_sitepackages = misc.site_packages
             _is_local = misc.is_local
             misc.is_local = req_uninstall.is_local = self.is_local
+            misc.site_packages = paths["purelib"]
             yield
             misc.is_local = req_uninstall.is_local = _is_local
             pkg_resources.working_set = _old_ws
+            misc.site_packages = _old_sitepackages
 
     def is_local(self, path) -> bool:
         return normalize_path(path).startswith(
@@ -199,10 +238,10 @@ class Environment:
             with builder_class(ireq) as builder:
                 return builder.build(**kwargs)
 
-    def get_working_set(self) -> pkg_resources.Environment:
+    def get_working_set(self) -> WorkingSet:
         """Get the working set based on local packages directory."""
         paths = self.get_paths()
-        return pkg_resources.Environment(
+        return WorkingSet(
             [paths["platlib"]], python=get_python_version(self.python_executable)
         )
 

pdm/models/requirements.py

@@ -26,12 +26,11 @@ VCS_SCHEMA = ("git", "hg", "svn", "bzr")
 VCS_REQ = re.compile(
     rf"(?P<vcs>{'|'.join(VCS_SCHEMA)})\+" r"(?P<url>[^\s;]+)(?P<marker>[\t ]*;[^\n]+)?"
 )
-_PATH_START = r"(?:\.|/|[a-zA-Z]:[/\\])"
 FILE_REQ = re.compile(
     r"(?:(?P<url>\S+://[^\s;]+)|"
-    rf"(?P<path>{_PATH_START}(?:[^\s;]|\\ )*"
-    rf"|'{_PATH_START}(?:[^']|\\')*'"
-    rf"|\"{_PATH_START}(?:[^\"]|\\\")*\"))"
+    rf"(?P<path>(?:[^\s;]|\\ )*"
+    rf"|'(?:[^']|\\')*'"
+    rf"|\"(?:[^\"]|\\\")*\"))"
     r"(?P<marker>[\t ]*;[^\n]+)?"
 )
 
@@ -107,6 +106,9 @@ class Requirement:
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__} {self.name}>"
 
+    def __str__(self) -> str:
+        return self.as_line()
+
     @classmethod
     def from_req_dict(cls, name: str, req_dict: RequirementDict) -> "Requirement":
         # TODO: validate req_dict
@@ -381,19 +383,18 @@ def filter_requirements_with_extras(
 
 def parse_requirement(line: str, editable: bool = False) -> Requirement:
 
-    r = None
     m = VCS_REQ.match(line)
     if m is not None:
         r = VcsRequirement.parse(line, m.groupdict())
     else:
-        m = FILE_REQ.match(line)
-        if m is not None:
-            r = FileRequirement.parse(line, m.groupdict())
-    if r is None:
         try:
             r = NamedRequirement.parse(line)  # type: Requirement
         except RequirementParseError as e:
-            raise RequirementError(str(e)) from None
+            m = FILE_REQ.match(line)
+            if m is not None:
+                r = FileRequirement.parse(line, m.groupdict())
+            else:
+                raise RequirementError(str(e)) from None
         else:
             if r.url:
                 r = FileRequirement(name=r.name, url=r.url, extras=r.extras)

pdm/project/core.py

@@ -227,12 +227,20 @@ class Project:
                 )
             } or None
             result[identify(req)] = can
-        if section == "default" and self.meta.name:
+        if section in ("default", "__all__") and self.meta.name:
             result[safe_name(self.meta.name).lower()] = self.make_self_candidate()
         return result
 
     def get_content_hash(self, algo: str = "md5") -> str:
-        pyproject_content = tomlkit.dumps(self.tool_settings)
+        # Only calculate sources and dependencies sections. Otherwise lock file is
+        # considered as unchanged.
+        dump_data = {"sources": self.tool_settings.get("source", [])}
+        for section in self.iter_sections():
+            toml_section = (
+                "dependencies" if section == "default" else f"{section}-dependencies"
+            )
+            dump_data[toml_section] = self.tool_settings.get(toml_section)
+        pyproject_content = tomlkit.dumps(dump_data)
         hasher = hashlib.new(algo)
         hasher.update(pyproject_content.encode("utf-8"))
         return hasher.hexdigest()

pyproject.toml

 [tool.pdm]
 name = "pdm"
-version = "0.0.3"
+version = "0.0.4"
 description = "Python Development Master"
 author = "frostming <mianghong@gmail.com>"
 license = "MIT"
@@ -26,7 +26,6 @@ pip_shims = "*"
 pythonfinder = "*"
 tomlkit = "*"
 halo = "<1.0.0,>=0.0.28"
-crayons = "<1.0.0,>=0.3.0"
 
 [tool.pdm.dev-dependencies]
 pytest = "*"
@@ -36,10 +35,6 @@ pytest-mock = "*"
 [tool.pdm.cli]
 pdm = "pdm.cli.commands:cli"
 
-[build-system]
-requires = ["intreehooks"]
-build-backend = "intreehooks:loader"
-
 [tool.intreehooks]
 build-backend = "pdm.builders.api"
 
@@ -77,3 +72,7 @@ known_third_party = "pip_shims"
 
 [tool.black]
 line-length = 88
+[build-system]
+requires = ["intreehooks"]
+build-backend = "intreehooks:loader"
+

tests/models/test_requirements.py

@@ -79,7 +79,7 @@ def test_convert_req_dict_to_req_line(req, req_dict, result):
 @pytest.mark.parametrize(
     "line,expected",
     [
-        ("requests; os_name=>'nt'", "Parse error at \"'; os_nam"),
+        ("requests; os_name=>'nt'", "Invalid marker:"),
         ("./nonexist", r"The local path (.+)? does not exist"),
         ("./tests", r"The local path (.+)? is not installable"),
     ],