Skip to content

P
Pdm

后端镜像 / Pdm

通知 0
Star 0
Fork 0
P
Pdm

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 e5f03b7d 编写于 作者: F Frost Ming
浏览文件
操作

Change for pip 19.3.1

上级 3a79ab89
隐藏空白更改
内联 并排
Showing with 174 addition and 153 deletion
pdm/context.py
浏览文件 @ e5f03b7d
import hashlib
from functools import wraps
from pathlib import Path
from pdm.exceptions import ProjectNotInitialized
from pdm.models.caches import CandidateInfoCache
from pdm.models.caches import HashCache
from pip_shims import shims
from pdm.exceptions import ProjectNotInitialized
from pdm.models.caches import CandidateInfoCache, HashCache
def require_initialize(func):
@wraps(func)
......
pdm/models/caches.py
浏览文件 @ e5f03b7d
import hashlib
import json
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from typing import Dict
from typing import Optional
from pip._vendor import requests
from typing import TYPE_CHECKING, Any, Dict, Optional
import pip_shims
from pip._vendor import requests
from pdm.exceptions import CorruptedCacheError
from pdm.types import CandidateInfo
from pdm.utils import unified_open_file
if TYPE_CHECKING:
from pdm.models.candidates import Candidate
......
pdm/models/candidates.py
浏览文件 @ e5f03b7d
import os
import warnings
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from typing import TYPE_CHECKING
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from pkg_resources import safe_extra
from pip._vendor.pkg_resources import safe_extra
from pip_shims import shims
from distlib.database import EggInfoDistribution
from distlib.metadata import Metadata
from distlib.wheel import Wheel
from pdm.context import context
from pdm.exceptions import ExtrasError
from pdm.exceptions import RequirementError
from pdm.exceptions import WheelBuildError
from pdm.exceptions import ExtrasError, RequirementError, WheelBuildError
from pdm.models.markers import Marker
from pdm.models.requirements import Requirement
from pdm.models.requirements import filter_requirements_with_extras
from pdm.utils import cached_property
from pdm.utils import create_tracked_tempdir
from pip_shims import shims
from pdm.models.requirements import Requirement, filter_requirements_with_extras
from pdm.utils import _allow_all_wheels, cached_property, create_tracked_tempdir
if TYPE_CHECKING:
from pdm.models.repositories import BaseRepository
......@@ -31,7 +21,7 @@ vcs = shims.VcsSupport()
def get_sdist(ireq: shims.InstallRequirement) -> Optional[EggInfoDistribution]:
egg_info = ireq.egg_info_path
egg_info = ireq.metadata_directory
return EggInfoDistribution(egg_info) if egg_info else None
......@@ -99,7 +89,7 @@ class Candidate:
"Editable installation is only supported for "
"local directory and VCS location."
)
ireq.run_egg_info()
ireq.prepare_metadata()
sdist = get_sdist(ireq)
self.metadata = sdist.metadata if sdist else None
else:
......@@ -135,7 +125,9 @@ class Candidate:
"""A local candidate has already everything in local, no need to download."""
kwargs = self._make_pip_wheel_args()
with self.repository.get_finder() as finder:
self.ireq.populate_link(finder, False, False)
with _allow_all_wheels():
# temporarily allow all wheels to get a link.
self.ireq.populate_link(finder, False, False)
if not self.req.editable and not self.req.name:
self.ireq.source_dir = kwargs["build_dir"]
else:
......@@ -161,9 +153,9 @@ class Candidate:
if not self.req.name:
# Name is not available for a tarball distribution. Get the package name
# from package's egg info.
# `run_egg_info()` won't work if there is a `req` attribute available.
# `prepare_metadata()` won't work if there is a `req` attribute available.
self.ireq.req = None
self.ireq.run_egg_info()
self.ireq.prepare_metadata()
self.req.name = self.ireq.metadata["Name"]
self.ireq.req = self.req
......@@ -173,9 +165,7 @@ class Candidate:
finder=finder, session=finder.session, **kwargs
) as preparer:
wheel_cache = context.make_wheel_cache()
builder = shims.WheelBuilder(
finder=finder, preparer=preparer, wheel_cache=wheel_cache
)
builder = shims.WheelBuilder(preparer=preparer, wheel_cache=wheel_cache)
output_dir = create_tracked_tempdir(prefix="pdm-ephem")
wheel_path = builder._build_one(self.ireq, output_dir)
if not wheel_path or not os.path.exists(wheel_path):
......@@ -237,7 +227,9 @@ class Candidate:
try:
requires_python = self.metadata.requires_python
except AttributeError:
requires_python = self.metadata._legacy.requires_python
requires_python = getattr(
self.metadata._legacy, "requires_python", "UNKNOWN"
)
if not requires_python or requires_python == "UNKNOWN":
requires_python = ""
if requires_python.isdigit():
......@@ -249,7 +241,7 @@ class Candidate:
"name": self.name,
"section": self.req.from_section,
"version": str(self.version),
"extras": self.req.extras,
"extras": sorted(self.req.extras or ()),
"marker": str(self.marker) if self.marker else None,
"editable": self.req.editable,
}
......@@ -257,5 +249,4 @@ class Candidate:
result.update({self.req.vcs: self.req.url, "revision": self.revision})
if not self.req.is_named:
result.update(url=self.req.url)
result.update(hashes=self.hashes)
return {k: v for k, v in result.items() if v}
pdm/models/markers.py
浏览文件 @ e5f03b7d
import copy
import operator
from functools import reduce
from typing import Iterable
from typing import Optional
from typing import Tuple
from typing import Union
from typing import Any, Iterable, Optional, Tuple, Union
from pip._vendor.packaging.markers import Marker as PackageMarker
......@@ -20,7 +16,7 @@ class Marker(PackageMarker):
return inst
def __and__(self, other: Optional[PackageMarker]) -> "Marker":
if other is None:
if other is None or self == other:
return self
lhs = f"({self})" if "or" in self._markers else str(self)
rhs = f"({other})" if "or" in other._markers else str(other)
......@@ -28,7 +24,7 @@ class Marker(PackageMarker):
return type(self)(marker_str)
def __rand__(self, other: Optional[PackageMarker]) -> "Marker":
if other is None:
if other is None or self == other:
return self
rhs = f"({self})" if "or" in self._markers else str(self)
lhs = f"({other})" if "or" in other._markers else str(other)
......@@ -36,17 +32,22 @@ class Marker(PackageMarker):
return type(self)(marker_str)
def __or__(self, other: Optional[PackageMarker]) -> "Marker":
if other is None:
if other is None or self == other:
return self
marker_str = f"{self} or {other}"
return type(self)(marker_str)
def __ror__(self, other: Optional[PackageMarker]) -> "Marker":
if other is None:
if other is None or self == other:
return self
marker_str = f"{other} or {self}"
return type(self)(marker_str)
def __eq__(self, other: Any) -> bool:
if not isinstance(other, PackageMarker):
return False
return str(self) == str(other)
def split_pyspec(self) -> Tuple[Optional["Marker"], PySpecSet]:
"""Split `python_version` and `python_full_version` from marker string"""
if _only_contains_python_keys(self._markers):
......@@ -134,7 +135,18 @@ def _build_pyspec_from_marker(markers):
op = ">="
elif op == "==":
version += ".*"
pyspec = PySpecSet(f"{op}{version}")
elif op in ("in", "not in"):
version = " ".join(v + ".*" for v in version.split())
if op == "in":
pyspec = reduce(
operator.or_, (PySpecSet(f"=={v}") for v in version.split())
)
elif op == "not in":
pyspec = reduce(
operator.and_, (PySpecSet(f"!={v}") for v in version.split())
)
else:
pyspec = PySpecSet(f"{op}{version}")
groups[-1] = groups[-1] & pyspec
else:
assert marker in ("and", "or")
......
pdm/models/repositories.py
浏览文件 @ e5f03b7d
import sys
from contextlib import contextmanager
from functools import wraps
from typing import Callable
from typing import List
from typing import Optional
from typing import Tuple
from typing import Callable, List, Optional, Tuple
import pip_shims
from pdm.context import context
from pdm.exceptions import CandidateInfoNotFound
from pdm.exceptions import CorruptedCacheError
from pdm.exceptions import CandidateInfoNotFound, CorruptedCacheError
from pdm.models.candidates import Candidate
from pdm.models.requirements import Requirement
from pdm.models.requirements import filter_requirements_with_extras
from pdm.models.specifiers import PySpecSet
from pdm.models.specifiers import SpecifierSet
from pdm.types import CandidateInfo
from pdm.types import Source
from pdm.utils import _allow_all_wheels
from pdm.utils import get_finder
from pdm.models.requirements import Requirement, filter_requirements_with_extras
from pdm.models.specifiers import PySpecSet, SpecifierSet
from pdm.types import CandidateInfo, Source
from pdm.utils import _allow_all_wheels, get_finder
def cache_result(
......@@ -208,7 +199,7 @@ class PyPIRepository(BaseRepository):
if allow_prereleases is None:
allow_prereleases = requirement.allow_prereleases
with self.get_finder(sources) as finder:
with self.get_finder(sources) as finder, _allow_all_wheels():
cans = [
Candidate.from_installation_candidate(c, requirement, self)
for c in finder.find_all_candidates(requirement.project_name)
......@@ -219,7 +210,7 @@ class PyPIRepository(BaseRepository):
for c in cans
if requirement.specifier.contains(c.version, allow_prereleases)
),
key=lambda c: c.version,
key=lambda c: (c.version, c.is_wheel),
)
if not allow_all:
sorted_cans = [
......
pdm/models/requirements.py
浏览文件 @ e5f03b7d
......@@ -2,37 +2,25 @@ import os
import re
import urllib.parse as urlparse
import warnings
from pathlib import Path
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Sequence
from typing import Tuple
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
import pip_shims
from pip._vendor.packaging.markers import InvalidMarker
from pip._vendor.pkg_resources import Requirement as PackageRequirement
from pip._vendor.pkg_resources import RequirementParseError
from pip._vendor.pkg_resources import safe_name
import pip_shims
from pip._vendor.pkg_resources import RequirementParseError, safe_name
from pip_shims import path_to_url, url_to_path
from pdm.exceptions import ExtrasError
from pdm.exceptions import RequirementError
from pdm.models.markers import Marker
from pdm.models.markers import get_marker
from pdm.models.markers import split_marker_element
from pdm.exceptions import ExtrasError, RequirementError
from pdm.models.markers import Marker, get_marker, split_marker_element
from pdm.models.readers import SetupReader
from pdm.models.specifiers import PySpecSet
from pdm.models.specifiers import get_specifier
from pdm.models.specifiers import PySpecSet, get_specifier
from pdm.types import RequirementDict
from pdm.utils import is_readonly_property
from pdm.utils import parse_name_version_from_wheel
from pdm.utils import url_without_fragments
from pip_shims import path_to_url
from pip_shims import url_to_path
from pdm.utils import (
is_readonly_property,
parse_name_version_from_wheel,
url_without_fragments,
)
VCS_SCHEMA = ("git", "hg", "svn", "bzr")
......@@ -94,6 +82,8 @@ class Requirement:
self.project_name = safe_name(self.name)
self.key = self.project_name.lower()
self.from_section = "default"
self.marker_no_python = None # type: Optional[Marker]
self.requires_python = PySpecSet() # type: PySpecSet
@property
def marker(self) -> Optional[Marker]:
......@@ -101,9 +91,12 @@ class Requirement:
@marker.setter
def marker(self, value) -> None:
# TODO: strip python requires
try:
self._marker = get_marker(value)
m = self._marker = get_marker(value)
if not m:
self.marker_no_python, self.requires_python = None, PySpecSet()
else:
self.marker_no_python, self.requires_python = m.split_pyspec()
except InvalidMarker as e:
raise RequirementError("Invalid marker: %s" % str(e)) from None
......@@ -375,20 +368,30 @@ class VcsRequirement(FileRequirement):
def filter_requirements_with_extras(
requirment_lines: List[str], extras: Sequence[str]
requirment_lines: List[Union[str, Dict[str, Union[str, List[str]]]]],
extras: Sequence[str],
) -> List[str]:
result = []
extras_in_meta = []
for req in requirment_lines:
_r = Requirement.from_line(req)
if not _r.marker:
result.append(req)
if isinstance(req, dict):
if req.get("extra"):
extras_in_meta.append(req["extra"])
if not req.get("extra") or req.get("extra") in extras:
marker = f"; {req['environment']}" if req.get("environment") else ""
result.extend(f"{line}{marker}" for line in req.get("requires", []))
else:
elements, rest = split_marker_element(str(_r.marker), "extra")
extras_in_meta.extend(e[1] for e in elements)
_r.marker = rest
if not elements or any(extra == e[1] for extra in extras for e in elements):
result.append(_r.as_line())
_r = Requirement.from_line(req)
if not _r.marker:
result.append(req)
else:
elements, rest = split_marker_element(str(_r.marker), "extra")
extras_in_meta.extend(e[1] for e in elements)
_r.marker = rest
if not elements or any(
extra == e[1] for extra in extras for e in elements
):
result.append(_r.as_line())
extras_not_found = [e for e in extras if e not in extras_in_meta]
if extras_not_found:
......
pdm/models/specifiers.py
浏览文件 @ e5f03b7d
import re
from functools import lru_cache
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Union
from typing import List, Optional, Set, Tuple, Union
from pip._vendor.packaging.specifiers import SpecifierSet
......@@ -20,11 +17,13 @@ def get_specifier(version_str: Union[SpecifierSet, str]) -> SpecifierSet:
def _parse_version_tuple(version: str) -> Tuple[Union[int, str], ...]:
version = re.sub(r"(?<!\.)\*", ".*", version)
try:
return tuple(int(v) if v != "*" else v for v in version.split("."))
except ValueError:
raise InvalidPyVersion(
"Prereleases or postreleases are not supported for python version"
f"{version}: Prereleases or postreleases are not supported "
"for python version specifers."
)
......
pdm/resolver/__init__.py
浏览文件 @ e5f03b7d
import copy
import time
from pdm.models.markers import PySpecSet
from pdm.models.markers import join_metaset
import tomlkit
from pdm.models.markers import PySpecSet, join_metaset
from pdm.models.requirements import Requirement
from pdm.resolver.providers import RepositoryProvider
from pdm.resolver.reporters import SimpleReporter
from resolvelib import Resolver
from vistir.contextmanagers import atomic_open_for_write
def _trace_visit_vertex(graph, current, target, visited, path, paths):
......@@ -62,7 +64,7 @@ def _build_marker_and_pyspec(dependencies, pythons, key, trace, all_metasets):
for parent, parent_metaset in all_parent_metasets.items():
r = dependencies[parent][key]
python = pythons[key]
marker, pyspec = r.marker.split_pyspec() if r.marker else (None, PySpecSet())
marker, pyspec = r.marker_no_python, r.requires_python
pyspec = python & pyspec
# Use 'and' to connect markers inherited from parent.
child_marker = (
......@@ -102,19 +104,46 @@ def _calculate_markers_and_pyspecs(traces, dependencies, pythons):
def format_lockfile(mapping, fetched_dependencies, summary_collection):
result = []
packages = tomlkit.aot()
metadata = tomlkit.table()
for k, v in mapping.items():
base = v.as_lockfile_entry()
deps = dict(r.as_req_dict() for r in fetched_dependencies[k].values())
new_data = {"summary": summary_collection[k], **base, "dependencies": deps}
result.append(new_data)
return result
base = tomlkit.table()
base.update(v.as_lockfile_entry())
base.add("summary", summary_collection[k])
deps = tomlkit.table()
for r in fetched_dependencies[k].values():
name, req = r.as_req_dict()
if getattr(req, "items", None) is not None:
inline = tomlkit.inline_table()
inline.update(req)
deps.add(name, inline)
else:
deps.add(name, req)
base.add("dependencies", deps)
packages.append(base)
if v.hashes:
key = f"{k} {v.version}"
metadata.add(key, tomlkit.array())
for filename, hash_value in v.hashes.items():
inline = tomlkit.inline_table()
inline.update({"file": filename, "hash": hash_value})
metadata[key].append(inline)
doc = tomlkit.document()
doc.update({"package": packages, "metadata": metadata})
return doc
def write_lockfile(toml_data):
with atomic_open_for_write("pdm.lock") as fp:
fp.write(tomlkit.dumps(toml_data))
def lock(requirements, repository, requires_python, allow_prereleases):
reqs = [Requirement.from_line(line) for line in requirements]
provider = RepositoryProvider(repository, requires_python, allow_prereleases)
reporter = SimpleReporter(reqs)
start = time.time()
resolver = Resolver(provider, reporter)
state = resolver.resolve(reqs)
provider.fetched_dependencies[None] = {provider.identify(r): r for r in reqs}
......@@ -138,5 +167,7 @@ def lock(requirements, repository, requires_python, allow_prereleases):
data = format_lockfile(
state.mapping, provider.fetched_dependencies, provider.summary_collection
)
write_lockfile(data)
print("total time cost: {} s".format(time.time() - start))
return data
pdm/resolver/providers.py
浏览文件 @ e5f03b7d
from typing import Dict
from typing import List
from typing import Optional
from typing import Union
from typing import Dict, List, Optional, Union
from pdm.models.candidates import Candidate
from pdm.models.repositories import BaseRepository
......@@ -49,7 +46,7 @@ class RepositoryProvider(AbstractProvider):
)
def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool:
if not candidate.version:
if not candidate.version or not requirement.is_named:
return True
return requirement.specifier.contains(
candidate.version
......@@ -57,8 +54,21 @@ class RepositoryProvider(AbstractProvider):
def get_dependencies(self, candidate: Candidate) -> List[Requirement]:
deps, requires_python, summary = self.repository.get_dependencies(candidate)
# Filter out incompatible dependencies(e.g. functools32) early so that
# we don't get errors when building wheels.
valid_deps = [
dep
for dep in deps
if not (
dep.requires_python & requires_python & self.requires_python
).is_impossible
]
candidate_key = self.identify(candidate.req)
self.fetched_dependencies[candidate_key] = {self.identify(r): r for r in deps}
self.fetched_dependencies[candidate_key] = {
self.identify(r): r for r in valid_deps
}
self.summary_collection[candidate_key] = summary
self.requires_python_collection[candidate_key] = requires_python
return deps
return valid_deps
pdm/resolver/reporters.py
浏览文件 @ e5f03b7d
......@@ -32,5 +32,5 @@ class SimpleReporter(BaseReporter):
"""Called before the resolution ends successfully.
"""
print("End resolving...")
elapsed = time.time() - self.start_at
print("Cost time:", elapsed)
# elapsed = time.time() - self.start_at
# print("Cost time:", elapsed)
pdm/utils.py
浏览文件 @ e5f03b7d
......@@ -8,24 +8,15 @@ import os
import shutil
import tempfile
import urllib.parse as parse
from contextlib import contextmanager
from typing import TYPE_CHECKING
from typing import Any
from typing import List
from typing import Optional
from typing import Tuple
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from distlib.wheel import Wheel
from pdm.types import Source
from pip_shims import Wheel as PipWheel
from pip_shims.backports import get_session
from pip_shims.backports import resolve_possible_shim
from pip_shims.shims import InstallCommand
from pip_shims.shims import PackageFinder
from pip_shims.shims import TargetPython
from pip_shims.shims import url_to_path
from pip_shims.backports import get_session, resolve_possible_shim
from pip_shims.shims import InstallCommand, PackageFinder, TargetPython, url_to_path
from distlib.wheel import Wheel
from pdm.types import Source
if TYPE_CHECKING:
from pip_shims.backports import (
......
pyproject.toml
浏览文件 @ e5f03b7d
[tool.isort]
line_length = 88
force_single_line = true
atomic = true
include_trailing_comma = true
lines_after_imports = 2
lines_between_types = 1
multi_line_output = 3
include_trailing_comma = true
use_parentheses = true
not_skip = "__init__.py"
skip_glob = ["*/setup.py"]
filter_files = true
known_first_party = "pdm"
known_third_party = "pip_shims"
setup.cfg
浏览文件 @ e5f03b7d
......@@ -5,5 +5,5 @@ exclude =
env,
dist,
build
test_script.py
temp_script.py
max_line_length = 88
test_script.py → temp_script.py
浏览文件 @ e5f03b7d
import json
import logging
from pdm.context import context
from pdm.models.candidates import Candidate
......@@ -8,6 +9,8 @@ from pdm.models.specifiers import PySpecSet
from pdm.resolver import lock
from resolvelib import Resolver
logging.getLogger("pip").setLevel(logging.INFO)
class FakeProject:
config = {"cache_dir": "./caches"}
......@@ -17,8 +20,12 @@ class FakeProject:
context.init(FakeProject())
source = {"url": "https://pypi.org/simple", "index": "pypi", "verify_ssl": True}
source = {
# "url": "https://pypi.org/simple",
"url": "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple",
"index": "pypi",
"verify_ssl": True,
}
repo = PyPIRepository([source])
data = lock(["tensorflow"], repo, FakeProject.python_requires, False)
json.dumps(data, indent=2)
data = lock(["fastapi[all]", "jupyterlab"], repo, FakeProject.python_requires, False)
tests/models/test_requirements.py
浏览文件 @ e5f03b7d
import os
import pytest
from pdm.models.requirements import Requirement
from pdm.models.requirements import RequirementError
from pdm.models.requirements import Requirement, RequirementError
from tests import FIXTURES
FILE_PREFIX = "{FILE_PREFIX}" if os.name == "nt" else "file://"
REQUIREMENTS = [
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或