diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index 0ff9b2fb7c..8df48fe7c8 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -20,6 +20,7 @@ import sys from pathlib import Path from typing import Optional, Tuple +import multiprocessing import click import piptools.writer as piptools_writer @@ -97,6 +98,7 @@ def main( extra_args: Tuple[str, ...], ) -> None: bazel_runfiles = runfiles.Create() + runfiles_dir = Path(bazel_runfiles._python_runfiles_root) / "_main" requirements_file = _select_golden_requirements_file( requirements_txt=requirements_txt, @@ -182,10 +184,16 @@ def main( resolved_requirements_file, requirements_file_tree ) ) - cli(argv) + # Run in a process as pip tools is calling sys.exit() + piptools_process = multiprocessing.Process(target=cli, args=[argv]) + piptools_process.start() + piptools_process.join() requirements_file_relative_path = Path(requirements_file_relative) content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") + # For windows or linux. + content = content.replace(str(runfiles_dir) + "/", "") + content = content.replace(str(runfiles_dir) + "\\", "") requirements_file_relative_path.write_text(content) else: # cli will exit(0) on success diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index dc5b186a6a..885aebf76e 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -27,6 +27,7 @@ def pip_compile( src = None, extra_args = [], extra_deps = [], + extra_data = [], generate_hashes = True, py_binary = _py_binary, py_test = _py_test, @@ -66,6 +67,7 @@ def pip_compile( [PEP621](https://peps.python.org/pep-0621/). extra_args: passed to pip-compile. extra_deps: extra dependencies passed to pip-compile. + extra_data: extra data passed to pip-compile. generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. py_test: the py_test rule to be used. @@ -99,7 +101,7 @@ def pip_compile( visibility = visibility, ) - data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + extra_data # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 2f4bcd6916..03736e5fef 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -35,7 +35,7 @@ def _get_annotation(requirement): name = requirement.split(" ")[0].split("=")[0].split("[")[0] return _annotations.get(name) -def install_deps(**whl_library_kwargs): +def install_deps(whl_files = {}, **whl_library_kwargs): """Repository rule macro. Install dependencies from `pip_parse`. Args: @@ -61,12 +61,16 @@ def install_deps(**whl_library_kwargs): for name, requirement in _packages: group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", "")) group_deps = all_requirement_groups.get(group_name, []) - + if "@ file://" in requirement: + whl_file = whl_files[name] + else: + whl_file = None whl_library( name = name, requirement = requirement, group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), + whl_file = whl_file, **whl_config )