feat: only consider Python versions that match a lock file's requires…

…-python set. (#149) With this change, pycross will use an imported lock file's `requires-python` (or equivalent) specifier set to further filter the list of target Python environments. For example, if Python 3.8 is in the full list of target environments, but the imported lock file specifies '>=3.9', Python 3.8 environments will be filtered out.
jvolkman · Jan 27, 2025 · fd48f76 · fd48f76
1 parent 656a226
commit fd48f76
Show file tree

Hide file tree

Showing 23 changed files with 72 additions and 107 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 
+### Added
+
+- When importing a lock file, only consider Python versions that match the lock file's
+  `requires-python` (or equivalent) set.
+
 ## [0.7.1]
 
 ### Added

diff --git a/e2e/pdm/always_build/MODULE.bazel b/e2e/pdm/always_build/MODULE.bazel
@@ -35,12 +35,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/pdm/build_wheel/MODULE.bazel b/e2e/pdm/build_wheel/MODULE.bazel
@@ -37,12 +37,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/pdm/local_wheel/MODULE.bazel b/e2e/pdm/local_wheel/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/pdm/requirements/MODULE.bazel b/e2e/pdm/requirements/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/poetry/always_build/MODULE.bazel b/e2e/poetry/always_build/MODULE.bazel
@@ -35,12 +35,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/poetry/build_wheel/MODULE.bazel b/e2e/poetry/build_wheel/MODULE.bazel
@@ -37,12 +37,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/poetry/local_wheel/MODULE.bazel b/e2e/poetry/local_wheel/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/poetry/requirements/MODULE.bazel b/e2e/poetry/requirements/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/uv/always_build/MODULE.bazel b/e2e/uv/always_build/MODULE.bazel
@@ -35,12 +35,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/uv/build_wheel/MODULE.bazel b/e2e/uv/build_wheel/MODULE.bazel
@@ -37,12 +37,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/uv/local_wheel/MODULE.bazel b/e2e/uv/local_wheel/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/e2e/uv/requirements/MODULE.bazel b/e2e/uv/requirements/MODULE.bazel
@@ -34,12 +34,6 @@ environments.create_for_python_toolchains(
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-gnu",
     ],
-    python_versions = [
-        "3.10.11",
-        "3.11.6",
-        "3.12.0",
-        "3.12",
-    ],
 )
 use_repo(environments, "rules_pycross_e2e_environments")
 

diff --git a/pycross/private/tools/lock_model.py b/pycross/private/tools/lock_model.py
@@ -15,6 +15,7 @@
 
 from dacite.config import Config
 from dacite.core import from_dict
+from packaging.specifiers import SpecifierSet
 from packaging.utils import canonicalize_name
 from packaging.utils import NormalizedName
 from packaging.utils import parse_sdist_filename
@@ -33,7 +34,7 @@ def _is_empty(val):
                 return len(val) == 0
             return False
 
-        if isinstance(o, (FileKey, PackageKey, Version)):
+        if isinstance(o, (FileKey, PackageKey, SpecifierSet, Version)):
             return str(o)
         if dataclasses.is_dataclass(o):
             # Omit None values from serialized output.
@@ -201,7 +202,7 @@ def key(self) -> PackageKey:
 class RawPackage:
     name: NormalizedName
     version: Version
-    python_versions: str
+    python_versions: SpecifierSet
     dependencies: List[PackageDependency] = field(default_factory=list)
     files: List[PackageFile] = field(default_factory=list)
 
@@ -211,7 +212,7 @@ def __post_init__(self):
         object.__setattr__(self, "name", normalized_name)
 
         assert self.version, "The version field must be specified."
-        assert self.python_versions is not None, "The python_versions field must be specified, or an empty string."
+        assert self.python_versions is not None, "The python_versions field must be specified."
         assert self.dependencies is not None, "The dependencies field must be specified as a list."
         assert self.files, "The files field must not be empty."
 
@@ -234,9 +235,13 @@ class ResolvedPackage:
 
 @dataclass(frozen=True)
 class RawLockSet:
+    python_versions: SpecifierSet
     packages: Dict[PackageKey, RawPackage] = field(default_factory=dict)
     pins: Dict[NormalizedName, PackageKey] = field(default_factory=dict)
 
+    def __post_init__(self):
+        assert self.python_versions is not None, "The python_versions field must be specified."
+
     @property
     def __dict__(self) -> Dict[str, Any]:
         return dict(_dataclass_items(self), packages=_stringify_keys(self.packages))
@@ -247,7 +252,7 @@ def to_json(self, indent=None) -> str:
     @classmethod
     def from_json(cls, data: str) -> RawLockSet:
         parsed = json.loads(data)
-        return from_dict(RawLockSet, parsed, config=Config(cast=[Tuple, Version, PackageKey]))
+        return from_dict(RawLockSet, parsed, config=Config(cast=[Tuple, Version, PackageKey, SpecifierSet]))
 
 
 @dataclass(frozen=True)

diff --git a/pycross/private/tools/pdm_translator.py b/pycross/private/tools/pdm_translator.py
@@ -43,21 +43,26 @@ class MismatchedVersionException(Exception):
 EDITABLE_PATTERN = re.compile("^ *-e +")
 
 
-def get_default_dependencies(lock: Dict[str, Any]) -> List[Requirement]:
-    deps = lock.get("project", {}).get("dependencies", [])
+def get_default_dependencies(project: Dict[str, Any]) -> List[Requirement]:
+    deps = project.get("project", {}).get("dependencies", [])
     return [Requirement(dep) for dep in deps]
 
 
-def get_optional_dependencies(lock: Dict[str, Any]) -> Dict[str, List[Requirement]]:
-    dep_groups = lock.get("project", {}).get("optional-dependencies", {})
+def get_optional_dependencies(project: Dict[str, Any]) -> Dict[str, List[Requirement]]:
+    dep_groups = project.get("project", {}).get("optional-dependencies", {})
     return {group: [Requirement(dep) for dep in deps] for group, deps in dep_groups.items()}
 
 
-def get_development_dependencies(lock: Dict[str, Any]) -> Dict[str, List[Requirement]]:
-    dep_groups = lock.get("tool", {}).get("pdm", {}).get("dev-dependencies", {})
+def get_development_dependencies(project: Dict[str, Any]) -> Dict[str, List[Requirement]]:
+    dep_groups = project.get("tool", {}).get("pdm", {}).get("dev-dependencies", {})
     return {group: [Requirement(EDITABLE_PATTERN.sub("", dep)) for dep in deps] for group, deps in dep_groups.items()}
 
 
+def get_requires_python(project: Dict[str, Any]) -> SpecifierSet:
+    requires_python = project.get("project", {}).get("requires-python", "")
+    return SpecifierSet(requires_python)
+
+
 def _print_warn(msg):
     print("WARNING:", msg)
 
@@ -97,7 +102,7 @@ def to_lock_package(self) -> RawPackage:
         return RawPackage(
             name=self.name,
             version=self.version,
-            python_versions=str(self.python_versions),
+            python_versions=self.python_versions,
             dependencies=dependencies_without_self,
             files=sorted(self.files, key=lambda f: f.name),
         )
@@ -178,6 +183,7 @@ def translate(
     default_dependencies = get_default_dependencies(project_dict)
     optional_dependencies = get_optional_dependencies(project_dict)
     development_dependencies = get_development_dependencies(project_dict)
+    lock_python_versions = get_requires_python(project_dict)
 
     if default_group:
         requirements.extend(default_dependencies)
@@ -296,6 +302,7 @@ def translate(
         lock_packages[lock_package.key] = lock_package
 
     return RawLockSet(
+        python_versions=lock_python_versions,
         packages=lock_packages,
         pins=pinned_keys,
     )

diff --git a/pycross/private/tools/poetry_translator.py b/pycross/private/tools/poetry_translator.py
@@ -7,6 +7,7 @@
 from typing import Optional
 
 import tomli
+from packaging.specifiers import SpecifierSet
 from packaging.utils import InvalidSdistFilename
 from packaging.utils import InvalidWheelFilename
 from packaging.utils import NormalizedName
@@ -55,7 +56,7 @@ def matches(self, other: "PoetryPackage") -> bool:
 class PoetryPackage:
     name: NormalizedName
     version: PoetryVersion
-    python_versions: str
+    python_versions: SpecifierSet
     dependencies: List[PoetryDependency]
     files: List[PackageFile]
     resolved_dependencies: List[PackageDependency]
@@ -78,6 +79,12 @@ def to_lock_package(self) -> RawPackage:
         )
 
 
+def parse_python_versions(python_versions: str) -> SpecifierSet:
+    if python_versions == "*":
+        return SpecifierSet()
+    return SpecifierSet(python_versions)
+
+
 def get_files_for_package(
     files: List[PackageFile],
     package_name: NormalizedName,
@@ -150,6 +157,9 @@ def parse_file_info(file_info) -> PackageFile:
         assert file_hash.startswith("sha256:")
         return PackageFile(name=file_name, sha256=file_hash[7:])
 
+    # Grab the list of supported Python versions
+    lock_python_versions = parse_python_versions(lock_dict.get("metadata", {}).get("python-versions", ""))
+
     # First, build a list of package files.
     # There are scenarios when files for multiple versions of a package are present in the list. They'll be filtered
     # later.
@@ -166,10 +176,6 @@ def parse_file_info(file_info) -> PackageFile:
         package_version = lock_pkg["version"]
         package_python_versions = lock_pkg["python-versions"]
 
-        if package_python_versions == "*":
-            # Special case for all python versions
-            package_python_versions = ""
-
         dependencies = []
         for name, dep_list in lock_pkg.get("dependencies", {}).items():
             # In some cases the dependency is actually a list of alternatives, each with a different
@@ -197,7 +203,7 @@ def parse_file_info(file_info) -> PackageFile:
             PoetryPackage(
                 name=package_name,
                 version=PoetryVersion.parse(package_version),
-                python_versions=package_python_versions,
+                python_versions=parse_python_versions(package_python_versions),
                 dependencies=dependencies,
                 files=get_files_for_package(
                     files,
@@ -252,6 +258,7 @@ def parse_file_info(file_info) -> PackageFile:
         lock_packages[lock_package.key] = lock_package
 
     return RawLockSet(
+        python_versions=lock_python_versions,
         packages=lock_packages,
         pins=pinned_keys,
     )