perf(resolver): Speed up the resolution with lazy find_matches (#1098)

news/1098.feature.md

+Lazy load the candidates returned by `find_matches()` to speed up the resolution.

pdm/resolver/providers.py

 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable, cast
 
 from packaging.specifiers import InvalidSpecifier, SpecifierSet
 from resolvelib import AbstractProvider
@@ -65,7 +65,7 @@ class BaseProvider(AbstractProvider):
         candidates: dict[str, Iterator[Candidate]],
         information: dict[str, Iterator[RequirementInformation]],
         backtrack_causes: Sequence[RequirementInformation],
-    ) -> Comparable:
+    ) -> tuple[Comparable, ...]:
         is_top = any(parent is None for _, parent in information[identifier])
         is_backtrack_cause = any(
             requirement.identify() == identifier
@@ -134,20 +134,24 @@ class BaseProvider(AbstractProvider):
         identifier: str,
         requirements: Mapping[str, Iterator[Requirement]],
         incompatibilities: Mapping[str, Iterator[Candidate]],
-    ) -> Iterable[Candidate]:
-        incompat = list(incompatibilities[identifier])
-        if identifier == "python":
-            candidates = find_python_matches(identifier, requirements)
-            return [c for c in candidates if c not in incompat]
-        elif identifier in self.overrides:
-            return self.get_override_candidates(identifier)
-        reqs = sorted(requirements[identifier], key=self.requirement_preference)
-        candidates = self._find_candidates(reqs[0])
-        return [
-            can
-            for can in candidates
-            if can not in incompat and all(self.is_satisfied_by(r, can) for r in reqs)
-        ]
+    ) -> Callable[[], Iterator[Candidate]]:
+        def matches_gen() -> Iterator[Candidate]:
+            incompat = list(incompatibilities[identifier])
+            if identifier == "python":
+                candidates = find_python_matches(identifier, requirements)
+                return (c for c in candidates if c not in incompat)
+            elif identifier in self.overrides:
+                return iter(self.get_override_candidates(identifier))
+            reqs = sorted(requirements[identifier], key=self.requirement_preference)
+            candidates = self._find_candidates(reqs[0])
+            return (
+                can
+                for can in candidates
+                if can not in incompat
+                and all(self.is_satisfied_by(r, can) for r in reqs)
+            )
+
+        return matches_gen
 
     def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool:
         if isinstance(requirement, PythonRequirement):
@@ -156,15 +160,19 @@ class BaseProvider(AbstractProvider):
             return True
         if not requirement.is_named:
             return not candidate.req.is_named and url_without_fragments(
-                candidate.req.url
-            ) == url_without_fragments(requirement.url)
-        version = candidate.version or candidate.metadata.version
+                candidate.req.url  # type: ignore
+            ) == url_without_fragments(
+                requirement.url  # type: ignore
+            )
+        version = candidate.version
         # Allow prereleases if: 1) it is not specified in the tool settings or
         # 2) the candidate doesn't come from PyPI index.
         allow_prereleases = (
             self.allow_prereleases in (True, None) or not candidate.req.is_named
         )
-        return requirement.specifier.contains(version, allow_prereleases)
+        return cast(SpecifierSet, requirement.specifier).contains(
+            version, allow_prereleases
+        )
 
     def get_dependencies(self, candidate: Candidate) -> list[Requirement]:
         if isinstance(candidate, PythonCandidate):
@@ -222,20 +230,28 @@ class ReusePinProvider(BaseProvider):
         identifier: str,
         requirements: Mapping[str, Iterator[Requirement]],
         incompatibilities: Mapping[str, Iterator[Candidate]],
-    ) -> Iterable[Candidate]:
+    ) -> Callable[[], Iterator[Candidate]]:
+        super_find = super().find_matches(identifier, requirements, incompatibilities)
         bare_name = strip_extras(identifier)[0]
-        if bare_name not in self.tracked_names and identifier in self.preferred_pins:
-            pin = self.preferred_pins[identifier]
-            incompat = list(incompatibilities[identifier])
-            demanded_req = next(requirements[identifier], None)
-            if demanded_req and demanded_req.is_named:
-                pin.req = demanded_req
-            pin._preferred = True
-            if pin not in incompat and all(
-                self.is_satisfied_by(r, pin) for r in requirements[identifier]
+
+        def matches_gen() -> Iterator[Candidate]:
+            if (
+                bare_name not in self.tracked_names
+                and identifier in self.preferred_pins
             ):
-                yield pin
-        yield from super().find_matches(identifier, requirements, incompatibilities)
+                pin = self.preferred_pins[identifier]
+                incompat = list(incompatibilities[identifier])
+                demanded_req = next(requirements[identifier], None)
+                if demanded_req and demanded_req.is_named:
+                    pin.req = demanded_req
+                pin._preferred = True  # type: ignore
+                if pin not in incompat and all(
+                    self.is_satisfied_by(r, pin) for r in requirements[identifier]
+                ):
+                    yield pin
+            yield from super_find()
+
+        return matches_gen
 
 
 class EagerUpdateProvider(ReusePinProvider):
@@ -276,7 +292,7 @@ class EagerUpdateProvider(ReusePinProvider):
         candidates: dict[str, Iterator[Candidate]],
         information: dict[str, Iterator[RequirementInformation]],
         backtrack_causes: Sequence[RequirementInformation],
-    ) -> Comparable:
+    ) -> tuple[Comparable, ...]:
         # Resolve tracking packages so we have a chance to unpin them first.
         (python, *others) = super().get_preference(
             identifier, resolutions, candidates, information, backtrack_causes

setup.cfg

@@ -46,4 +46,4 @@ ignore_missing_imports = True
 disallow_incomplete_defs = True
 disallow_untyped_defs = True
 disallow_untyped_decorators = True
-exclude = pdm/(pep582/|models/in_process/.+\.py|resolver/providers\.py)
+exclude = pdm/(pep582/|models/in_process/.+\.py)