synchronize dependencies function

.github/workflows/.ci.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install -r requirements.txt
-        python -m pdm install
+        python -m pdm install -d
     - name: Test
       run: |
         python -m pdm run pytest --cov pdm tests

LICENSE

 MIT License
 
-Copyright (c) 2019 Frost Ming
+Copyright (c) 2019-2020 Frost Ming
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

pdm/cli/actions.py

+from typing import Optional, Tuple
+
+from pdm.installers import Synchronizer
+from pdm.project import Project
+
+
+def do_sync(
+    project: Project, sections: Tuple[str, ...], dev: bool,
+    default: bool, dry_run: bool, clean: Optional[bool]
+) -> None:
+    """Synchronize project
+
+    :param project: The project instance.
+    :param sections: A tuple of optional sections to be synced.
+    :param dev: whether to include dev-dependecies.
+    :param default: whether to include default dependencies.
+    :param dry_run: Print actions without actually running them.
+    :param clean: whether to remove unneeded packages.
+    """
+    clean = default if clean is None else clean
+    candidates = {}
+    for section in sections:
+        candidates.update(project.get_locked_candidates(section))
+    if dev:
+        candidates.update(project.get_locked_candidates("dev"))
+    if default:
+        candidates.update(project.get_locked_candidates())
+    handler = Synchronizer(candidates, project.environment)
+    handler.synchronize(clean=clean, dry_run=dry_run)

pdm/cli/commands.py

@@ -3,6 +3,8 @@ import shutil
 import subprocess
 
 import click
+from pdm.cli import actions
+from pdm.exceptions import CommandNotFound
 from pdm.project import Project
 from pdm.resolver import lock as _lock
 
@@ -36,18 +38,17 @@ def lock(project):
     "--no-default", "default", flag_value=False, default=True,
     help="Don't install dependencies from default seciton."
 )
+@click.option(
+    "--no-lock", "lock", flag_value=False, default=True,
+    help="Don't do lock if lockfile is not found or outdated."
+)
 @pass_project
-def install(project, sections, dev, default):
-    candidates = set()
-    if default:
-        candidates.update(project.get_locked_candidates())
-    if dev:
-        candidates.update(project.get_locked_candidates("dev"))
-    for section in sections:
-        candidates.update(project.get_locked_candidates(section))
-    installer = project.get_installer()
-    for can in candidates:
-        installer.install_candidate(can)
+def install(project, sections, dev, default, lock):
+    if lock and not (
+        project.lockfile_file.is_file() and project.is_lockfile_hash_match()
+    ):
+        _lock(project)
+    actions.do_sync(project, sections, dev, default, False, False)
 
 
 @cli.command(
@@ -59,5 +60,31 @@ def install(project, sections, dev, default):
 @pass_project
 def run(project, command, args):
     with project.environment.activate():
-        command = shutil.which(command, path=os.getenv("PATH"))
-        subprocess.run([command] + list(args))
+        expanded_command = shutil.which(command, path=os.getenv("PATH"))
+        if not expanded_command:
+            raise CommandNotFound(command)
+        subprocess.run([expanded_command] + list(args))
+
+
+@cli.command(help="Synchronizes current working set with lock file.")
+@click.option(
+    "-s", "--section", "sections", multiple=True, help="Specify section(s) to install."
+)
+@click.option(
+    "-d", "--dev", default=False, is_flag=True, help="Also install dev dependencies."
+)
+@click.option(
+    "--no-default", "default", flag_value=False, default=True,
+    help="Don't install dependencies from default seciton."
+)
+@click.option(
+    "--dry-run", is_flag=True, default=False,
+    help="Only prints actions without actually running them."
+)
+@click.option(
+    "--clean/--no-clean", "clean", default=None,
+    help="Whether to remove unneeded packages from working set."
+)
+@pass_project
+def sync(project, sections, dev, default, dry_run, clean):
+    actions.do_sync(project, sections, dev, default, dry_run, clean)

pdm/context.py

+import hashlib
+from functools import wraps
+from pathlib import Path
+
+from pip_shims import shims
+
+from pdm.exceptions import ProjectNotInitialized
+from pdm.models.caches import CandidateInfoCache, HashCache
+
+
+def require_initialize(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if not self.initialized:
+            raise ProjectNotInitialized()
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class Context:
+    """A singleton context object that holds some global states.
+    Global states are evil but make it easier to share configs between
+    different modules.
+    """
+
+    def __init__(self):
+        self.project = None
+        self._initialized = False
+
+    def init(self, project):
+        self.project = project
+        self._initialized = True
+
+    @property
+    def initialized(self) -> bool:
+        return self._initialized
+
+    @property
+    @require_initialize
+    def cache_dir(self) -> Path:
+        return Path(self.project.config.get("cache_dir"))
+
+    @require_initialize
+    def cache(self, name: str) -> Path:
+        path = self.cache_dir / name
+        path.mkdir(parents=True, exist_ok=True)
+        return path
+
+    @require_initialize
+    def make_wheel_cache(self) -> shims.WheelCache:
+        return shims.WheelCache(
+            self.cache_dir.as_posix(), shims.FormatControl(set(), set()),
+        )
+
+    @require_initialize
+    def make_candidate_info_cache(self) -> CandidateInfoCache:
+        python_hash = hashlib.sha1(
+            str(self.project.python_requires).encode()
+        ).hexdigest()
+        file_name = f"package_meta_{python_hash}.json"
+        return CandidateInfoCache(self.cache_dir / file_name)
+
+    @require_initialize
+    def make_hash_cache(self) -> HashCache:
+        return HashCache(directory=self.cache("hashes").as_posix())
+
+
+context = Context()

pdm/exceptions.py

@@ -40,7 +40,7 @@ class ExtrasError(UserWarning):
         return f"Extras not found: {self.extras}"
 
 
-class NoProjectError(PdmException):
+class ProjectError(PdmException):
     pass
 
 
@@ -79,3 +79,12 @@ class RequirementsConflicted(ResolutionError):
 
 class NoPythonVersion(PdmException):
     pass
+
+
+class CommandNotFound(PdmException):
+    def __init__(self, command):
+        super.__init__(command)
+        self.command = command
+
+    def __str__(self):
+        return f"'{self.command}' is not found in your PATH."

pdm/installers.py

 import os
 import subprocess
+from typing import Dict, List, Tuple
 
+from pip._vendor.pkg_resources import Distribution, WorkingSet
 from pip_shims import shims
 
 import distlib.scripts
@@ -17,13 +19,21 @@ SETUPTOOLS_SHIM = (
 )
 
 
+def _is_dist_editable(working_set: WorkingSet, dist: Distribution) -> bool:
+    for entry in working_set.entries:
+        if os.path.isfile(os.path.join(entry, dist.project_name + ".egg-link")):
+            return True
+    return False
+
+
 class Installer:
     # TODO: Support PEP 517 builds
 
-    def __init__(self, environment: Environment) -> None:
+    def __init__(self, environment: Environment, auto_confirm: bool = True) -> None:
         self.environment = environment
+        self.auto_confirm = auto_confirm
 
-    def install_candidate(self, candidate: Candidate) -> None:
+    def install(self, candidate: Candidate) -> None:
         print(f"Installing {candidate.name} {candidate.version}...")
         candidate.get_metadata()
         if candidate.wheel:
@@ -58,3 +68,87 @@ class Installer:
                 subprocess.check_call(install_args)
         finally:
             os.chdir(old_pwd)
+
+    def uninstall(self, name: str) -> None:
+        working_set = self.environment.get_working_set()
+        ireq = shims.install_req_from_line(name)
+        print(f"Uninstalling: {name} {working_set.by_key[name].version}")
+
+        with self.environment.activate():
+            pathset = ireq.uninstall(auto_confirm=self.auto_confirm)
+            if pathset:
+                pathset.commit()
+
+
+class Synchronizer:
+    """Synchronize the working set with given installation candidates"""
+
+    def __init__(
+        self,
+        candidates: Dict[str, Candidate],
+        environment: Environment,
+    ) -> None:
+        self.candidates = candidates
+        self.environment = environment
+
+    def get_installer(self) -> Installer:
+        return Installer(self.environment)
+
+    def compare_with_working_set(self) -> Tuple[List[str], List[str], List[str]]:
+        """Compares the candidates and return (to_add, to_update, to_remove)"""
+        working_set = self.environment.get_working_set()
+        to_update, to_remove = [], []
+        candidates = self.candidates.copy()
+        for dist in working_set:
+            if dist.key not in candidates:
+                to_remove.append(dist.key)
+            else:
+                can = candidates.pop(dist.key)
+                if (
+                    not _is_dist_editable(working_set, dist)
+                    and dist.version != can.version
+                ):
+                    # XXX: An editable distribution is always considered as consistent.
+                    to_update.append(dist.key)
+        to_add = list(candidates)
+        return to_add, to_update, to_remove
+
+    def install_candidates(self, candidates: List[Candidate]) -> None:
+        installer = self.get_installer()
+        for can in candidates:
+            installer.install(can)
+
+    def remove_distributions(self, distributions: List[str]) -> None:
+        installer = self.get_installer()
+        for name in distributions:
+            installer.uninstall(name)
+
+    def synchronize(self, clean: bool = True, dry_run: bool = False) -> None:
+        """Synchronize the working set with pinned candidates.
+
+        :param clean: Whether to remove unneeded packages, defaults to True.
+        :param dry_run: If set to True, only prints actions without actually do them.
+        """
+        to_add, to_update, to_remove = self.compare_with_working_set()
+        lists_to_check = [to_add, to_update]
+        if clean:
+            lists_to_check.append(to_remove)
+        if not any(lists_to_check):
+            print("All packages are synced to date, nothing to do.")
+            return
+        if to_add:
+            print("Packages to be added:", ", ".join(to_add))
+            if not dry_run:
+                self.install_candidates(
+                    [can for k, can in self.candidates.items() if k in to_add]
+                )
+        if to_update:
+            print("Packages to be updated:", ", ".join(to_update))
+            if not dry_run:
+                self.install_candidates(
+                    [can for k, can in self.candidates.items() if k in to_update]
+                )
+        if to_remove:
+            print("Packages to be removed:", ", ".join(to_remove))
+            if not dry_run:
+                self.remove_distributions(to_remove)

pdm/models/candidates.py

@@ -49,7 +49,6 @@ class Candidate:
         self._requires_python = None
 
         self.wheel = None
-        self.build_dir = None
         self.metadata = None
 
     def __hash__(self):

pdm/models/environment.py

-import hashlib
+from __future__ import annotations
+
 import os
 import sys
 import sysconfig
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
+from pip._internal.req import req_uninstall
+from pip._internal.utils import misc
+from pip._vendor import pkg_resources
 from pip_shims import shims
 
-from distlib.wheel import Wheel
+from pdm.context import context
 from pdm.exceptions import NoPythonVersion, WheelBuildError
-from pdm.models.caches import CandidateInfoCache, HashCache
-from pdm.models.specifiers import PySpecSet
-from pdm.project.config import Config
-from pdm.types import Source
 from pdm.utils import (
     _allow_all_wheels, cached_property, convert_hashes, create_tracked_tempdir, get_finder, get_python_version,
 )
 from pythonfinder import Finder
 from vistir.contextmanagers import temp_environ
+from vistir.path import normalize_path
+
+if TYPE_CHECKING:
+    from pdm.models.specifiers import PySpecSet
+    from distlib.wheel import Wheel
+    from pdm.project.config import Config
+    from pdm.types import Source
 
 
 class Environment:
@@ -26,30 +33,6 @@ class Environment:
         self.python_requires = python_requires
         self.config = config
 
-    @property
-    def cache_dir(self) -> Path:
-        return Path(self.config.get("cache_dir"))
-
-    def cache(self, name: str) -> Path:
-        path = self.cache_dir / name
-        path.mkdir(parents=True, exist_ok=True)
-        return path
-
-    def make_wheel_cache(self) -> shims.WheelCache:
-        return shims.WheelCache(
-            self.cache_dir.as_posix(), shims.FormatControl(set(), set()),
-        )
-
-    def make_candidate_info_cache(self) -> CandidateInfoCache:
-        python_hash = hashlib.sha1(
-            str(self.python_requires).encode()
-        ).hexdigest()
-        file_name = f"package_meta_{python_hash}.json"
-        return CandidateInfoCache(self.cache_dir / file_name)
-
-    def make_hash_cache(self) -> HashCache:
-        return HashCache(directory=self.cache("hashes").as_posix())
-
     @cached_property
     def python_executable(self) -> str:
         """Get the Python interpreter path."""
@@ -98,7 +81,21 @@ class Environment:
             os.environ["PATH"] = os.pathsep.join(
                 [python_root, paths["scripts"], os.environ["PATH"]]
             )
+            working_set = self.get_working_set()
+            _old_ws = pkg_resources.working_set
+            pkg_resources.working_set = working_set
+            # HACK: Replace the is_local with environment version so that packages can
+            # be removed correctly.
+            _is_local = misc.is_local
+            misc.is_local = req_uninstall.is_local = self.is_local
             yield
+            misc.is_local = req_uninstall.is_local = _is_local
+            pkg_resources.working_set = _old_ws
+
+    def is_local(self, path) -> bool:
+        return normalize_path(path).startswith(
+            normalize_path(self.packages_path.as_posix())
+        )
 
     @cached_property
     def packages_path(self) -> Path:
@@ -120,8 +117,8 @@ class Environment:
             build_dir = src_dir
         else:
             build_dir = create_tracked_tempdir(prefix="pdm-build")
-        download_dir = self.cache("pkgs")
-        wheel_download_dir = self.cache("wheels")
+        download_dir = context.cache("pkgs")
+        wheel_download_dir = context.cache("wheels")
         return {
             "build_dir": build_dir,
             "src_dir": src_dir,
@@ -152,7 +149,7 @@ class Environment:
         python_version = get_python_version(self.python_executable)[:2]
         finder = get_finder(
             sources,
-            self.cache_dir.as_posix(),
+            context.cache_dir.as_posix(),
             python_version,
             ignore_requires_python,
         )
@@ -192,7 +189,7 @@ class Environment:
 
             if ireq.link.is_wheel:
                 return Wheel(
-                    (self.cache("wheels") / ireq.link.filename).as_posix()
+                    (context.cache("wheels") / ireq.link.filename).as_posix()
                 )
             # VCS url is unpacked, now build the egg-info
             if ireq.editable and ireq.req.is_vcs:
@@ -212,10 +209,14 @@ class Environment:
             with shims.make_preparer(
                 finder=finder, session=finder.session, **kwargs
             ) as preparer:
-                wheel_cache = self.make_wheel_cache()
+                wheel_cache = context.make_wheel_cache()
                 builder = shims.WheelBuilder(preparer=preparer, wheel_cache=wheel_cache)
                 output_dir = create_tracked_tempdir(prefix="pdm-ephem")
                 wheel_path = builder._build_one(ireq, output_dir)
                 if not wheel_path or not os.path.exists(wheel_path):
                     raise WheelBuildError(str(ireq))
                 return Wheel(wheel_path)
+
+    def get_working_set(self) -> pkg_resources.WorkingSet:
+        paths = self.get_paths()
+        return pkg_resources.WorkingSet([paths["platlib"]])

pdm/models/repositories.py

@@ -4,6 +4,7 @@ import sys
 from functools import wraps
 from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple
 
+from pdm.context import context
 from pdm.exceptions import CandidateInfoNotFound, CorruptedCacheError
 from pdm.models.candidates import Candidate
 from pdm.models.requirements import Requirement, filter_requirements_with_extras, parse_requirement
@@ -31,8 +32,8 @@ class BaseRepository:
     def __init__(self, sources: List[Source], environment: Environment) -> None:
         self.sources = sources
         self.environment = environment
-        self._candidate_info_cache = self.environment.make_candidate_info_cache()
-        self._hash_cache = self.environment.make_hash_cache()
+        self._candidate_info_cache = context.make_candidate_info_cache()
+        self._hash_cache = context.make_hash_cache()
 
     def get_filtered_sources(self, req: Requirement) -> List[Source]:
         if not req.index:

pdm/project/__init__.py

@@ -4,7 +4,8 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 import tomlkit
-from pdm.installers import Installer
+from pdm.context import context
+from pdm.exceptions import ProjectError
 from pdm.models.candidates import Candidate
 from pdm.models.environment import Environment
 from pdm.models.repositories import BaseRepository, PyPIRepository
@@ -63,6 +64,7 @@ class Project:
         self._pyproject = None  # type: Optional[TOMLDocument]
         self._lockfile = None  # type: Optional[TOMLDocument]
         self._config = None  # type: Optional[Config]
+        context.init(self)
 
     def __repr__(self) -> str:
         return f"<Project '{self.root.as_posix()}'>"
@@ -80,7 +82,9 @@ class Project:
     @property
     def lockfile(self):
         # type: () -> TOMLDocument
-        if not self._lockfile and self.lockfile_file.exists():
+        if not self.lockfile_file.is_file():
+            raise ProjectError("Lock file does not exist.")
+        if not self._lockfile:
             data = tomlkit.parse(self.lockfile_file.read_text("utf-8"))
             self._lockfile = data
         return self._lockfile
@@ -168,9 +172,11 @@ class Project:
             fp.write(tomlkit.dumps(toml_data))
         self._lockfile = None
 
-    def get_locked_candidates(self, section: Optional[str] = None) -> List[Candidate]:
+    def get_locked_candidates(
+        self, section: Optional[str] = None
+    ) -> Dict[str, Candidate]:
         section = section or "default"
-        result = []
+        result = {}
         for package in [dict(p) for p in self.lockfile["package"]]:
             if section not in package["sections"]:
                 continue
@@ -188,7 +194,7 @@ class Project:
                     f"{package_name} {version}", []
                 )
             }
-            result.append(can)
+            result[req.key] = can
         return result
 
     def is_lockfile_hash_match(self) -> bool:
@@ -200,6 +206,3 @@ class Project:
         pyproject_content = tomlkit.dumps(self.pyproject)
         content_hash = hasher(pyproject_content.encode("utf-8")).hexdigest()
         return content_hash == hash_value
-
-    def get_installer(self) -> Installer:
-        return Installer(self.environment)

pdm/utils.py

@@ -20,7 +20,7 @@ from pip_shims.backports import get_session, resolve_possible_shim
 from pip_shims.shims import InstallCommand, PackageFinder, TargetPython
 
 from distlib.wheel import Wheel
-from pdm.exceptions import NoProjectError
+from pdm.exceptions import ProjectError
 from pdm.types import Source
 
 if TYPE_CHECKING:
@@ -314,7 +314,7 @@ def find_project_root(cwd: str = ".", max_depth: int = 5):
             break
         path = path.parent
 
-    raise NoProjectError(
+    raise ProjectError(
         f"No pyproject.toml is found from directory '{original_path.as_posix}'"
     )
 

temp_script.py

 import logging
 
+from pdm.installers import Installer
 from pdm.project import Project
-from pdm.resolver import lock
 
 project = Project()
 # lock(project)
-installer = project.get_installer()
-for can in project.get_locked_candidates():
-    installer.install_candidate(can)
+installer = Installer(project.environment)
+installer.uninstall("idna")