From d4ca501f743d41cacfdecfb7e88799baa90e246b Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Fri, 18 Dec 2020 17:36:21 +0800 Subject: [PATCH] reuse cached wheels --- news/200.feature | 1 + pdm/cli/actions.py | 8 +++--- pdm/cli/commands/show.py | 6 ++--- pdm/cli/utils.py | 8 +++--- pdm/formats/base.py | 11 ++++++--- pdm/formats/flit.py | 4 +-- pdm/formats/legacy.py | 12 ++++++--- pdm/formats/poetry.py | 10 ++++++-- pdm/models/environment.py | 18 +++++++++++++- pdm/models/{metadata.py => project_info.py} | 2 +- pdm/models/requirements.py | 27 ++++++++++++++++++++- pdm/project/core.py | 2 +- pdm/project/metadata.py | 4 ++- tests/cli/test_actions.py | 2 +- 14 files changed, 85 insertions(+), 30 deletions(-) create mode 100644 news/200.feature rename pdm/models/{metadata.py => project_info.py} (99%) diff --git a/news/200.feature b/news/200.feature new file mode 100644 index 00000000..d381f066 --- /dev/null +++ b/news/200.feature @@ -0,0 +1 @@ +Reuse the cached built wheels to accelerate the installation. diff --git a/pdm/cli/actions.py b/pdm/cli/actions.py index 39871bb8..0317c5ee 100644 --- a/pdm/cli/actions.py +++ b/pdm/cli/actions.py @@ -18,7 +18,7 @@ from pdm.cli.utils import ( ) from pdm.exceptions import NoPythonVersion, PdmUsageError, ProjectError from pdm.formats import FORMATS -from pdm.formats.base import array_of_inline_tables +from pdm.formats.base import array_of_inline_tables, make_inline_table from pdm.installers.installers import format_dist from pdm.iostream import LOCK, stream from pdm.models.builders import EnvBuilder @@ -362,24 +362,22 @@ def do_init( python_requires: str = "", ) -> None: """Bootstrap the project and create a pyproject.toml""" - import tomlkit - data = { "project": { "name": name, "version": version, "description": "", "authors": array_of_inline_tables([{"name": author, "email": email}]), - "license": tomlkit.inline_table({"text": license}), + "license": make_inline_table({"text": license}), "urls": {"homepage": ""}, "dependencies": [], "dev-dependencies": [], + "requires-python": python_requires, }, "build-system": {"requires": ["pdm-pep517"], "build-backend": "pdm.pep517.api"}, } if python_requires and python_requires != "*": get_specifier(python_requires) - data["requires-python"] = python_requires if not project.pyproject: project._pyproject = data else: diff --git a/pdm/cli/commands/show.py b/pdm/cli/commands/show.py index 3cb846c5..fe9f88b5 100644 --- a/pdm/cli/commands/show.py +++ b/pdm/cli/commands/show.py @@ -5,7 +5,7 @@ from pip._vendor.pkg_resources import safe_name from pdm.cli.commands.base import BaseCommand from pdm.iostream import stream from pdm.models.candidates import Candidate -from pdm.models.metadata import Metadata +from pdm.models.project_info import ProjectInfo from pdm.models.requirements import parse_requirement from pdm.project import Project @@ -43,9 +43,9 @@ class Command(BaseCommand): metadata = latest.get_metadata() if metadata._legacy: - result = Metadata(dict(metadata._legacy.items()), True) + result = ProjectInfo(dict(metadata._legacy.items()), True) else: - result = Metadata(dict(metadata._data), False) + result = ProjectInfo(dict(metadata._data), False) if latest_stable: result.latest_stable_version = str(latest_stable.version) if installed: diff --git a/pdm/cli/utils.py b/pdm/cli/utils.py index af02e02f..f828a204 100644 --- a/pdm/cli/utils.py +++ b/pdm/cli/utils.py @@ -13,6 +13,7 @@ from resolvelib.structs import DirectedGraph from pdm.exceptions import ProjectError from pdm.formats import FORMATS +from pdm.formats.base import make_inline_table from pdm.iostream import stream from pdm.models.environment import WorkingSet from pdm.models.requirements import Requirement, strip_extras @@ -332,9 +333,7 @@ def format_lockfile(mapping, fetched_dependencies, summary_collection): for r in fetched_dependencies[k].values(): name, req = r.as_req_dict() if getattr(req, "items", None) is not None: - inline = tomlkit.inline_table() - inline.update(req) - deps.add(name, inline) + deps.add(name, make_inline_table(req)) else: deps.add(name, req) if len(deps) > 0: @@ -345,8 +344,7 @@ def format_lockfile(mapping, fetched_dependencies, summary_collection): array = tomlkit.array() array.multiline(True) for filename, hash_value in v.hashes.items(): - inline = tomlkit.inline_table() - inline.update({"file": filename, "hash": hash_value}) + inline = make_inline_table({"file": filename, "hash": hash_value}) array.append(inline) if array: file_hashes.add(key, array) diff --git a/pdm/formats/base.py b/pdm/formats/base.py index 7fafb989..69908b7d 100644 --- a/pdm/formats/base.py +++ b/pdm/formats/base.py @@ -79,13 +79,18 @@ class MetaConverter(collections.abc.Mapping, metaclass=_MetaConverterMeta): NAME_EMAIL_RE = re.compile(r"(?P[^,]+?)\s*<(?P.+)>\s*$") +def make_inline_table(data): + """Create an inline table from the given data.""" + table = tomlkit.inline_table() + table.update(data) + return table + + def array_of_inline_tables(value, multiline=True): container = tomlkit.array() container.multiline(multiline) for item in value: - table = tomlkit.inline_table() - table.update(item) - container.append(table) + container.append(make_inline_table(item)) return container diff --git a/pdm/formats/flit.py b/pdm/formats/flit.py index 9de724bf..42a4dd82 100644 --- a/pdm/formats/flit.py +++ b/pdm/formats/flit.py @@ -4,7 +4,7 @@ from pathlib import Path import tomlkit import tomlkit.exceptions -from pdm.formats.base import MetaConverter, convert_from +from pdm.formats.base import MetaConverter, convert_from, make_inline_table def check_fingerprint(project, filename): @@ -46,7 +46,7 @@ class FlitMetaConverter(MetaConverter): if "maintainer" in metadata: self._data["maintainers"] = _get_author(metadata, "maintainer") if "license" in metadata: - self._data["license"] = {"text", metadata.pop("license")} + self._data["license"] = make_inline_table({"text", metadata.pop("license")}) if "urls" in metadata: self._data["urls"] = metadata.pop("urls") if "home-page" in metadata: diff --git a/pdm/formats/legacy.py b/pdm/formats/legacy.py index 537b402f..5c44ed70 100644 --- a/pdm/formats/legacy.py +++ b/pdm/formats/legacy.py @@ -3,7 +3,13 @@ import functools import tomlkit import tomlkit.exceptions -from pdm.formats.base import MetaConverter, Unset, convert_from, parse_name_email +from pdm.formats.base import ( + MetaConverter, + Unset, + convert_from, + make_inline_table, + parse_name_email, +) from pdm.models.requirements import Requirement @@ -38,9 +44,7 @@ class LegacyMetaConverter(MetaConverter): @convert_from("license") def license(self, value): - table = tomlkit.inline_table() - table["text"] = value - return table + return make_inline_table({"text": value}) @convert_from("source") def source(self, value): diff --git a/pdm/formats/poetry.py b/pdm/formats/poetry.py index 27dccee1..1e4ff2f0 100644 --- a/pdm/formats/poetry.py +++ b/pdm/formats/poetry.py @@ -5,7 +5,13 @@ import re import tomlkit import tomlkit.exceptions -from pdm.formats.base import MetaConverter, Unset, convert_from, parse_name_email +from pdm.formats.base import ( + MetaConverter, + Unset, + convert_from, + make_inline_table, + parse_name_email, +) from pdm.models.markers import Marker from pdm.models.requirements import Requirement from pdm.models.specifiers import PySpecSet @@ -82,7 +88,7 @@ class PoetryMetaConverter(MetaConverter): @convert_from("license") def license(self, value): - return {"text": value} + return make_inline_table({"text": value}) @convert_from(name="requires-python") def requires_python(self, source): diff --git a/pdm/models/environment.py b/pdm/models/environment.py index f8d38990..c4951d7a 100644 --- a/pdm/models/environment.py +++ b/pdm/models/environment.py @@ -313,10 +313,26 @@ class Environment: shutil.copy(downloaded.path, download_dir) except shutil.SameFileError: pass - # Now all source is prepared, build it. + if ireq.link.is_wheel: + # If the file is a wheel, should be already present under download dir. return (self.project.cache("wheels") / ireq.link.filename).as_posix() + else: + # Check the built wheel cache again after hashes are resolved. + cache_entry = wheel_cache.get_cache_entry( + ireq.link, + ireq.req.project_name, + pip_shims.get_supported( + version="".join( + map(str, get_python_version(self.python_executable)[:2]) + ) + ), + ) + if cache_entry is not None: + stream.logger.debug("Using cached wheel link: %s", cache_entry.link) + return cache_entry.link.file_path + # Otherwise, now all source is prepared, build it. with EnvBuilder(ireq.unpacked_source_directory, self) as builder: if ireq.editable: ret = builder.build_egg_info(kwargs["build_dir"]) diff --git a/pdm/models/metadata.py b/pdm/models/project_info.py similarity index 99% rename from pdm/models/metadata.py rename to pdm/models/project_info.py index 8ddd0008..98073ab7 100644 --- a/pdm/models/metadata.py +++ b/pdm/models/project_info.py @@ -3,7 +3,7 @@ from typing import Dict, Iterator, List, Tuple, Union from pdm.iostream import stream -class Metadata: +class ProjectInfo: def __init__(self, data: Dict[str, Union[str, List[str]]], legacy: bool) -> None: self._data = data self.legacy = legacy diff --git a/pdm/models/requirements.py b/pdm/models/requirements.py index 0fa06044..b6053829 100644 --- a/pdm/models/requirements.py +++ b/pdm/models/requirements.py @@ -3,7 +3,7 @@ import re import urllib.parse as urlparse import warnings from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union from pip._vendor.packaging.markers import InvalidMarker from pip._vendor.pkg_resources import Requirement as PackageRequirement @@ -173,6 +173,31 @@ class Requirement: def as_line(self) -> str: raise NotImplementedError + def as_req_dict(self) -> Tuple[str, RequirementDict]: + r = {} + if self.editable: + r["editable"] = True + if self.extras: + r["extras"] = sorted(self.extras) + if self.is_vcs: + r[self.vcs] = self.repo + elif self.path and self.is_local_dir: + r["path"] = self.str_path + elif self.url: + r["url"] = self.url + if self.marker: + r["marker"] = str(self.marker).replace('"', "'") + if self.specifier: + r["version"] = str(self.specifier) + elif self.is_named: + r["version"] = "*" + if len(r) == 1 and next(iter(r), None) == "version": + r = r["version"] + for attr in ["index", "allow_prereleases", "ref"]: + if getattr(self, attr) is not None: + r[attr] = getattr(self, attr) + return self.project_name, r + def matches(self, line: str) -> bool: """Return whether the passed in PEP 508 string is the same requirement as this one. diff --git a/pdm/project/core.py b/pdm/project/core.py index 5cdfaa02..0fcd5cf7 100644 --- a/pdm/project/core.py +++ b/pdm/project/core.py @@ -177,7 +177,7 @@ class Project: def iter_sections(self) -> Iterable[str]: if self.meta.dependencies: yield "default" - if self.meta.dev_dependencies: + if self.meta.get("dev-dependencies"): yield "dev" if self.meta.optional_dependencies: yield from self.meta.optional_dependencies.keys() diff --git a/pdm/project/metadata.py b/pdm/project/metadata.py index 1ab6cc88..0eb48d6f 100644 --- a/pdm/project/metadata.py +++ b/pdm/project/metadata.py @@ -11,7 +11,9 @@ class MutableMetadata(Metadata, MutableMapping): def __init__(self, filepath, data=None) -> None: self.filepath = filepath - self._metadata = data or self._read_pyproject(filepath) + if data is None: + data = self._read_pyproject(filepath) + self._metadata = data def __getitem__(self, k): return self._metadata[k] diff --git a/tests/cli/test_actions.py b/tests/cli/test_actions.py index f543c8eb..034c5970 100644 --- a/tests/cli/test_actions.py +++ b/tests/cli/test_actions.py @@ -281,7 +281,7 @@ def test_remove_package_exist_in_multi_section(project, repository, working_set) actions.do_add(project, packages=["requests"]) actions.do_add(project, dev=True, packages=["urllib3"]) actions.do_remove(project, dev=True, packages=["urllib3"]) - assert not any("urllib3" in line for line in project.meta.dev_dependencies) + assert not any("urllib3" in line for line in project.meta["dev-dependencies"]) assert "urllib3" in working_set assert "requests" in working_set -- GitLab