From d2ca9a382175e12bae2fe7c817691b1fdb0bde95 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 21 Jan 2021 11:48:53 +0800 Subject: [PATCH] Expand PROJECT_ROOT on build --- pdm/models/auth.py | 19 ++----------------- pdm/models/candidates.py | 11 +++++++---- pdm/models/environment.py | 13 ++++++++++--- pdm/models/requirements.py | 18 ++++++++++++++---- pdm/project/core.py | 15 ++++++++++----- pdm/utils.py | 19 +++++++++++++++++++ tests/models/test_candidates.py | 29 +++++++++++++++++++++++++++++ tests/test_utils.py | 6 ++++-- 8 files changed, 95 insertions(+), 35 deletions(-) diff --git a/pdm/models/auth.py b/pdm/models/auth.py index 9d8da0fd..f27cc531 100644 --- a/pdm/models/auth.py +++ b/pdm/models/auth.py @@ -1,32 +1,17 @@ -from typing import List, Optional, Tuple +from typing import List from pdm._types import Source from pdm.exceptions import PdmException from pdm.models.pip_shims import MultiDomainBasicAuth -from pdm.utils import expand_env_vars class PdmBasicAuth(MultiDomainBasicAuth): """A custom auth class that differs from Pip's implementation in the following ways: - 1. It expands env variables in URL auth. - 2. It shows an error message when credentials are not provided or correect. + - It shows an error message when credentials are not provided or correect. """ - def _get_url_and_credentials( - self, original_url: str - ) -> Tuple[str, Optional[str], Optional[str]]: - url, username, password = super()._get_url_and_credentials(original_url) - - if username: - username = expand_env_vars(username) - - if password: - password = expand_env_vars(password) - - return url, username, password - def handle_401(self, resp, **kwargs): if resp.status_code == 401 and not self.prompting: raise PdmException( diff --git a/pdm/models/candidates.py b/pdm/models/candidates.py index ce05cbaa..6d62a136 100644 --- a/pdm/models/candidates.py +++ b/pdm/models/candidates.py @@ -13,7 +13,7 @@ from pdm.iostream import stream from pdm.models import pip_shims from pdm.models.markers import Marker from pdm.models.requirements import Requirement, filter_requirements_with_extras -from pdm.utils import cached_property +from pdm.utils import cached_property, path_replace if TYPE_CHECKING: from distlib.metadata import Metadata @@ -230,13 +230,16 @@ class Candidate: "marker": str(self.marker).replace('"', "'") if self.marker else None, "editable": self.req.editable, } + project_root = self.environment.project.root.as_posix().lstrip("/") if self.req.is_vcs: result.update({self.req.vcs: self.req.repo, "revision": self.revision}) elif not self.req.is_named: - if self.req.path: - result.update(path=self.req.str_path) + if self.req.is_file_or_url and self.req.is_local_dir: + result.update(path=path_replace(project_root, ".", self.req.str_path)) else: - result.update(url=self.req.url) + result.update( + url=path_replace(project_root, "${PROJECT_ROOT}", self.req.url) + ) return {k: v for k, v in result.items() if v} def format(self) -> str: diff --git a/pdm/models/environment.py b/pdm/models/environment.py index cf89e4c3..87c64b93 100644 --- a/pdm/models/environment.py +++ b/pdm/models/environment.py @@ -258,7 +258,9 @@ class Environment: """ if sources is None: sources = self.project.sources - sources = sources or [] + for source in sources: + source["url"] = expand_env_vars_in_auth(source["url"]) + python_version, _ = get_python_version(self.python_executable, digits=2) finder = get_finder( sources, @@ -321,10 +323,15 @@ class Environment: only_download = True if hashes: ireq.hash_options = convert_hashes(hashes) + ireq.link = pip_shims.Link( + expand_env_vars_in_auth( + ireq.link.url.replace( + "${PROJECT_ROOT}", self.project.root.as_posix().lstrip("/") + ) + ) + ) if not (ireq.editable and ireq.req.is_local_dir): downloader = pip_shims.Downloader(finder.session, "off") - if ireq.link.is_vcs: - ireq.link = pip_shims.Link(expand_env_vars_in_auth(ireq.link.url)) downloaded = pip_shims.unpack_url( ireq.link, ireq.source_dir, diff --git a/pdm/models/requirements.py b/pdm/models/requirements.py index 7d8db1e1..900b0d93 100644 --- a/pdm/models/requirements.py +++ b/pdm/models/requirements.py @@ -243,7 +243,9 @@ class Requirement: class FileRequirement(Requirement): def __init__(self, **kwargs): super().__init__(**kwargs) - self.path = Path(self.path) if self.path else None + self.path = ( + Path(str(self.path).replace("${PROJECT_ROOT}", ".")) if self.path else None + ) self.version = None self._parse_url() if self.path and not self.path.exists(): @@ -276,10 +278,19 @@ class FileRequirement(Requirement): def _parse_url(self) -> None: if not self.url: if self.path: - self.url = path_to_url(self.path.as_posix()) + self.url = path_to_url( + self.path.as_posix().replace("${PROJECT_ROOT}", ".") + ) else: try: - self.path = Path(url_to_path(self.url)) + self.path = Path( + url_to_path( + self.url.replace( + "${PROJECT_ROOT}", + Path(".").absolute().as_posix().lstrip("/"), + ) + ) + ) except AssertionError: pass self._parse_name_from_url() @@ -361,7 +372,6 @@ class VcsRequirement(FileRequirement): def __init__(self, **kwargs): self.repo = None super().__init__(**kwargs) - self._parse_url() @classmethod def parse(cls, line: str, parsed: Dict[str, str]) -> "VcsRequirement": diff --git a/pdm/project/core.py b/pdm/project/core.py index b6474397..71693125 100644 --- a/pdm/project/core.py +++ b/pdm/project/core.py @@ -25,6 +25,7 @@ from pdm.project.metadata import MutableMetadata as Metadata from pdm.utils import ( atomic_open_for_write, cached_property, + cd, find_project_root, get_venv_python, setdefault, @@ -170,11 +171,15 @@ class Project: else: deps = metadata.get("optional-dependencies", {}).get(section, []) result = {} - for line in deps: - req = parse_requirement(line) - req.from_section = section or "default" - # make editable packages behind normal ones to override correctly. - result[req.identify()] = req + with cd(self.root): + for line in deps: + if line.startswith("-e "): + req = parse_requirement(line[3:].strip(), True) + else: + req = parse_requirement(line) + req.from_section = section or "default" + # make editable packages behind normal ones to override correctly. + result[req.identify()] = req return result @property diff --git a/pdm/utils.py b/pdm/utils.py index 01990458..028aa985 100644 --- a/pdm/utils.py +++ b/pdm/utils.py @@ -2,6 +2,7 @@ Utility functions """ import atexit +import functools import os import re import shutil @@ -378,3 +379,21 @@ def expand_env_vars_in_auth(url: str) -> str: auth = expand_env_vars(auth, True) netloc = "@".join([auth, rest]) return parse.urlunparse((scheme, netloc, path, params, query, fragment)) + + +@functools.lru_cache() +def path_replace(pattern: str, replace_with: str, dest: str) -> str: + """Safely replace the pattern in a path with given string. + + :param pattern: the pattern to match + :param replace_with: the string to replace with + :param dest: the path to replace + :return the replaced path + """ + sub_flags = re.IGNORECASE if os.name == "nt" else 0 + return re.sub( + pattern.replace("\\", "/"), + replace_with, + dest.replace("\\", "/"), + flags=sub_flags, + ) diff --git a/tests/models/test_candidates.py b/tests/models/test_candidates.py index 9ab641d7..7d966bed 100644 --- a/tests/models/test_candidates.py +++ b/tests/models/test_candidates.py @@ -3,6 +3,7 @@ import pytest from pdm.exceptions import ExtrasError from pdm.models.candidates import Candidate from pdm.models.requirements import parse_requirement +from pdm.project.core import Project from tests import FIXTURES @@ -93,3 +94,31 @@ def test_parse_abnormal_specifiers(project): ) candidate = Candidate(req, project.environment) assert candidate.get_dependencies_from_metadata() + + +@pytest.mark.parametrize( + "req_str", + [ + "demo @ file:///${PROJECT_ROOT}/tests/fixtures/artifacts" + "/demo-0.0.1-py2.py3-none-any.whl", + "demo @ file:///${PROJECT_ROOT}/tests/fixtures/artifacts/demo-0.0.1.tar.gz", + "demo @ file:///${PROJECT_ROOT}/tests/fixtures/projects/demo", + "-e ${PROJECT_ROOT}/tests/fixtures/projects/demo", + ], +) +def test_expand_project_root_in_url(req_str): + project = Project(FIXTURES.parent.parent) + if req_str.startswith("-e "): + req = parse_requirement(req_str[3:], True) + else: + req = parse_requirement(req_str) + candidate = Candidate(req, project.environment) + assert candidate.get_dependencies_from_metadata() == [ + "idna", + 'chardet; os_name == "nt"', + ] + lockfile_entry = candidate.as_lockfile_entry() + if "path" in lockfile_entry: + assert lockfile_entry["path"].startswith("./") + else: + assert "${PROJECT_ROOT}" in lockfile_entry["url"] diff --git a/tests/test_utils.py b/tests/test_utils.py index 7356dd21..ce2a2da1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -55,8 +55,10 @@ def test_find_python_in_path(tmp_path): cli_utils.find_python_in_path(sys.executable) == pathlib.Path(sys.executable).as_posix() ) - assert cli_utils.find_python_in_path(sys.prefix).startswith( - pathlib.Path(sys.executable).as_posix() + assert ( + cli_utils.find_python_in_path(sys.prefix) + .lower() + .startswith(pathlib.Path(sys.executable).as_posix().lower()) ) with pytest.raises(PdmException): cli_utils.find_python_in_path(tmp_path) -- GitLab