From fb86f6792dd3e4b896edf738312ac7684cd63496 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Fri, 10 Nov 2023 16:18:55 +0100 Subject: [PATCH] Allow downstream projects configs to be file_config (#1778) Reapply https://github.com/bazelbuild/continuous-integration/pull/1776 with fixes: - We need to properly clone the git repository so that we can get `file_config` at the correct commit. - Refactored `downstream-root` so that we don't need to get platform info from task config for cloning the repo, which introduces a chicken-egg problem. - Allow suppressing stdout in `clone_git_repository` so that it won't impact stdout of `project_pipeline` - Make `validate_config` also work for downstream projects by adding `git_commit` as a parameter of `get_modified_files`. --- buildkite/bazel-bench/bazel_bench.py | 4 +- .../bazel-central-registry/bcr_presubmit.py | 19 +- buildkite/bazel_auto_sheriff.py | 6 +- buildkite/bazelci.py | 241 +++++++----------- buildkite/culprit_finder.py | 40 +-- 5 files changed, 119 insertions(+), 191 deletions(-) diff --git a/buildkite/bazel-bench/bazel_bench.py b/buildkite/bazel-bench/bazel_bench.py index 1f8405f7d1..ef7b670308 100644 --- a/buildkite/bazel-bench/bazel_bench.py +++ b/buildkite/bazel-bench/bazel_bench.py @@ -77,7 +77,6 @@ AGGR_JSON_PROFILES_FILENAME = "aggr_json_profiles.csv" PLATFORMS_WHITELIST = ['macos', 'ubuntu1804'] REPORT_GENERATION_PLATFORM = 'ubuntu1804' -STARTER_JOB_PLATFORM = 'ubuntu1804' def _bazel_bench_env_setup_command(platform, bazel_commits): @@ -425,8 +424,7 @@ def main(args=None): else datetime.date.today() ) - bazel_clone_path = bazelci.clone_git_repository( - BAZEL_REPOSITORY, STARTER_JOB_PLATFORM) + bazel_clone_path = bazelci.clone_git_repository(BAZEL_REPOSITORY) bazel_commits_full_list, bazel_commits_to_benchmark = _get_bazel_commits( date, bazel_clone_path, parsed_args.max_commits) bazel_bench_ci_steps = [] diff --git a/buildkite/bazel-central-registry/bcr_presubmit.py b/buildkite/bazel-central-registry/bcr_presubmit.py index ad3ec5fe66..dd42ff76a0 100755 --- a/buildkite/bazel-central-registry/bcr_presubmit.py +++ b/buildkite/bazel-central-registry/bcr_presubmit.py @@ -154,16 +154,9 @@ def scratch_file(root, relative_path, lines=None, mode="w"): return abspath -def get_root_dir(module_name, module_version, task, is_test_module=False): - # TODO(pcloudy): We use the "downstream root" as the repo root, find a better root path for BCR presubmit. - configs = get_test_module_task_config(module_name, module_version) if is_test_module else get_task_config(module_name, module_version) - platform = bazelci.get_platform_for_task(task, configs["tasks"][task]) - return pathlib.Path(bazelci.downstream_projects_root(platform)) - - -def create_simple_repo(module_name, module_version, task): +def create_simple_repo(module_name, module_version): """Create a simple Bazel module repo which depends on the target module.""" - root = get_root_dir(module_name, module_version, task) + root = bazelci.get_repositories_root() scratch_file(root, "WORKSPACE") scratch_file(root, "BUILD") # TODO(pcloudy): Should we test this module as the root module? Maybe we do if we support dev dependency. @@ -217,10 +210,10 @@ def unpack_archive(archive_file, output_dir): else: shutil.unpack_archive(archive_file, output_dir) -def prepare_test_module_repo(module_name, module_version, task): +def prepare_test_module_repo(module_name, module_version): """Prepare the test module repo and the presubmit yml file it should use""" bazelci.print_collapsed_group(":information_source: Prepare test module repo") - root = get_root_dir(module_name, module_version, task, is_test_module = True) + root = bazelci.get_repositories_root() source = load_source_json(module_name, module_version) # Download and unpack the source archive to ./output @@ -420,11 +413,11 @@ def main(argv=None): pipeline_steps = [{"block": "Wait on BCR maintainer review", "blocked_state": "running"}] + pipeline_steps upload_jobs_to_pipeline(pipeline_steps) elif args.subparsers_name == "runner": - repo_location = create_simple_repo(args.module_name, args.module_version, args.task) + repo_location = create_simple_repo(args.module_name, args.module_version) config_file = get_presubmit_yml(args.module_name, args.module_version) return run_test(repo_location, config_file, args.task) elif args.subparsers_name == "test_module_runner": - repo_location, config_file = prepare_test_module_repo(args.module_name, args.module_version, args.task) + repo_location, config_file = prepare_test_module_repo(args.module_name, args.module_version) return run_test(repo_location, config_file, args.task) else: parser.print_help() diff --git a/buildkite/bazel_auto_sheriff.py b/buildkite/bazel_auto_sheriff.py index e1c2c75381..703a20e5b6 100644 --- a/buildkite/bazel_auto_sheriff.py +++ b/buildkite/bazel_auto_sheriff.py @@ -126,11 +126,7 @@ def _get_main_build_result(self): job_infos = filter(lambda x: bool(x), (extract_job_info_by_key(job) for job in main_build_info["jobs"] if not ("soft_failed" in job and job["soft_failed"]))) self.main_result["tasks"] = group_job_info_by_task(job_infos) self.main_result["state"] = get_project_state(self.main_result["tasks"]) - - last_green_commit_url = bazelci.bazelci_last_green_commit_url( - bazelci.DOWNSTREAM_PROJECTS[self.project]["git_repository"], self.pipeline - ) - self.main_result["last_green_commit"] = bazelci.get_last_green_commit(last_green_commit_url) + self.main_result["last_green_commit"] = bazelci.get_last_green_commit(self.project) # Log all succeeded projects in the same annotate block diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 14f02dfbea..8367f7f2d1 100755 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -85,91 +85,82 @@ DOWNSTREAM_PROJECTS_PRODUCTION = { "Android Studio Plugin": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/android-studio.yml", + "file_config": ".bazelci/android-studio.yml", "pipeline_slug": "android-studio-plugin", }, "Android Studio Plugin Google": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/google/.bazelci/android-studio.yml", + "file_config": ".bazelci/android-studio.yml", "pipeline_slug": "android-studio-plugin-google", }, "Android Testing": { "git_repository": "https://github.com/googlesamples/android-testing.git", - "http_config": "https://raw.githubusercontent.com/googlesamples/android-testing/master/bazelci/buildkite-pipeline.yml", + "file_config": "bazelci/buildkite-pipeline.yml", "pipeline_slug": "android-testing", "disabled_reason": "https://github.com/android/testing-samples/issues/417", }, "Bazel": { "git_repository": "https://github.com/bazelbuild/bazel.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/postsubmit.yml", + "file_config": ".bazelci/postsubmit.yml", "pipeline_slug": "bazel-bazel", }, "Bazel Bench": { "git_repository": "https://github.com/bazelbuild/bazel-bench.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-bench/master/.bazelci/postsubmit.yml", + "file_config": ".bazelci/postsubmit.yml", "pipeline_slug": "bazel-bench", }, "Bazel Codelabs": { "git_repository": "https://github.com/bazelbuild/codelabs.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/codelabs/master/.bazelci/presubmit.yml", "pipeline_slug": "bazel-codelabs", "disabled_reason": "https://github.com/bazelbuild/codelabs/issues/38", }, "Bazel Examples": { "git_repository": "https://github.com/bazelbuild/examples.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/examples/master/.bazelci/presubmit.yml", "pipeline_slug": "bazel-bazel-examples", }, "Bazel Remote Cache": { "git_repository": "https://github.com/buchgr/bazel-remote.git", - "http_config": "https://raw.githubusercontent.com/buchgr/bazel-remote/master/.bazelci/presubmit.yml", "pipeline_slug": "bazel-remote-cache", }, "Bazel skylib": { "git_repository": "https://github.com/bazelbuild/bazel-skylib.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-skylib/main/.bazelci/presubmit.yml", "pipeline_slug": "bazel-skylib", "owned_by_bazel": True, }, "Bazel toolchains": { "git_repository": "https://github.com/bazelbuild/bazel-toolchains.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-toolchains/master/.bazelci/presubmit.yml", "pipeline_slug": "bazel-toolchains", }, "Bazel watcher": { "git_repository": "https://github.com/bazelbuild/bazel-watcher.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-watcher/master/.bazelci/presubmit.yml", "pipeline_slug": "bazel-watcher", "disabled_reason": "https://github.com/bazelbuild/bazel-watcher/issues/590", }, "Bazelisk": { "git_repository": "https://github.com/bazelbuild/bazelisk.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/.bazelci/config.yml", + "file_config": ".bazelci/config.yml", "pipeline_slug": "bazelisk", }, "Buildfarm": { "git_repository": "https://github.com/bazelbuild/bazel-buildfarm.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/bazel-buildfarm/main/.bazelci/presubmit.yml", "pipeline_slug": "buildfarm-farmer", }, "Buildtools": { "git_repository": "https://github.com/bazelbuild/buildtools.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/buildtools/master/.bazelci/presubmit.yml", "pipeline_slug": "buildtools", }, "CLion Plugin": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/clion.yml", + "file_config": ".bazelci/clion.yml", "pipeline_slug": "clion-plugin", }, "CLion Plugin Google": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/google/.bazelci/clion.yml", + "file_config": ".bazelci/clion.yml", "pipeline_slug": "clion-plugin-google", }, "Cartographer": { "git_repository": "https://github.com/googlecartographer/cartographer.git", - "http_config": "https://raw.githubusercontent.com/googlecartographer/cartographer/master/.bazelci/presubmit.yml", "pipeline_slug": "cartographer", "disabled_reason": "https://github.com/cartographer-project/cartographer/issues/1938#issuecomment-1804795113", }, @@ -180,13 +171,11 @@ }, "Envoy": { "git_repository": "https://github.com/envoyproxy/envoy.git", - "http_config": "https://raw.githubusercontent.com/envoyproxy/envoy/main/.bazelci/presubmit.yml", "pipeline_slug": "envoy", "disabled_reason": "https://github.com/envoyproxy/envoy/issues/29723", }, "FlatBuffers": { "git_repository": "https://github.com/google/flatbuffers.git", - "http_config": "https://raw.githubusercontent.com/google/flatbuffers/master/.bazelci/presubmit.yml", "pipeline_slug": "flatbuffers", "disabled_reason": "https://github.com/google/flatbuffers/issues/7992", }, @@ -202,42 +191,40 @@ }, "Google Logging": { "git_repository": "https://github.com/google/glog.git", - "http_config": "https://raw.githubusercontent.com/google/glog/master/.bazelci/presubmit.yml", "pipeline_slug": "google-logging", }, "IntelliJ Plugin": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/intellij.yml", + "file_config": ".bazelci/intellij.yml", "pipeline_slug": "intellij-plugin", }, "IntelliJ Plugin Google": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/google/.bazelci/intellij.yml", + "file_config": ".bazelci/intellij.yml", "pipeline_slug": "intellij-plugin-google", }, "IntelliJ UE Plugin": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/intellij-ue.yml", + "file_config": ".bazelci/intellij-ue.yml", "pipeline_slug": "intellij-ue-plugin", }, "IntelliJ UE Plugin Google": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/google/.bazelci/intellij-ue.yml", + "file_config": ".bazelci/intellij-ue.yml", "pipeline_slug": "intellij-ue-plugin-google", }, "IntelliJ Plugin Aspect": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/master/.bazelci/aspect.yml", + "file_config": ".bazelci/aspect.yml", "pipeline_slug": "intellij-plugin-aspect", }, "IntelliJ Plugin Aspect Google": { "git_repository": "https://github.com/bazelbuild/intellij.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/intellij/google/.bazelci/aspect.yml", + "file_config": ".bazelci/aspect.yml", "pipeline_slug": "intellij-plugin-aspect-google", }, "Kythe": { "git_repository": "https://github.com/kythe/kythe.git", - "http_config": "https://raw.githubusercontent.com/kythe/kythe/master/.bazelci/presubmit.yml", "pipeline_slug": "kythe", }, "Protobuf": { @@ -248,17 +235,9 @@ }, "Stardoc": { "git_repository": "https://github.com/bazelbuild/stardoc.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/stardoc/master/.bazelci/presubmit.yml", "pipeline_slug": "stardoc", "owned_by_bazel": True, }, - "Subpar": { - "git_repository": "https://github.com/google/subpar.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/subpar.yml", - "pipeline_slug": "subpar", - "owned_by_bazel": True, - "disabled_reason": "https://github.com/google/subpar/issues/133", - }, "TensorFlow": { "git_repository": "https://github.com/tensorflow/tensorflow.git", "http_config": "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/pipelines/tensorflow.yml", @@ -272,154 +251,118 @@ }, "rules_android": { "git_repository": "https://github.com/bazelbuild/rules_android.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_android/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-android", }, "rules_android_ndk": { "git_repository": "https://github.com/bazelbuild/rules_android_ndk.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_android_ndk/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-android-ndk", }, - "rules_appengine": { - "git_repository": "https://github.com/bazelbuild/rules_appengine.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_appengine/master/.bazelci/presubmit.yml", - "pipeline_slug": "rules-appengine-appengine", - "disabled_reason": "https://github.com/bazelbuild/rules_appengine/issues/127", - }, "rules_apple": { "git_repository": "https://github.com/bazelbuild/rules_apple.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_apple/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-apple-darwin", }, "rules_cc": { "git_repository": "https://github.com/bazelbuild/rules_cc.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_cc/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-cc", "owned_by_bazel": True, }, "rules_closure": { "git_repository": "https://github.com/bazelbuild/rules_closure.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_closure/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-closure-closure-compiler", "owned_by_bazel": True, }, "rules_dotnet": { "git_repository": "https://github.com/bazelbuild/rules_dotnet.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_dotnet/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-dotnet-edge", }, "rules_foreign_cc": { "git_repository": "https://github.com/bazelbuild/rules_foreign_cc.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_foreign_cc/main/.bazelci/config.yaml", + "file_config": ".bazelci/config.yaml", "pipeline_slug": "rules-foreign-cc", "owned_by_bazel": True, }, "rules_go": { "git_repository": "https://github.com/bazelbuild/rules_go.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_go/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-go-golang", }, "rules_groovy": { "git_repository": "https://github.com/bazelbuild/rules_groovy.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_groovy/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-groovy", }, "rules_gwt": { "git_repository": "https://github.com/bazelbuild/rules_gwt.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_gwt/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-gwt", "disabled_reason": "https://github.com/bazelbuild/continuous-integration/issues/1202", }, "rules_haskell": { "git_repository": "https://github.com/tweag/rules_haskell.git", - "http_config": "https://raw.githubusercontent.com/tweag/rules_haskell/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-haskell-haskell", }, "rules_jsonnet": { "git_repository": "https://github.com/bazelbuild/rules_jsonnet.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jsonnet/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-jsonnet", }, "rules_jvm_external": { "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-jvm-external", "owned_by_bazel": True, }, "rules_jvm_external - examples": { "git_repository": "https://github.com/bazelbuild/rules_jvm_external.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_jvm_external/master/.bazelci/examples.yml", + "file_config": ".bazelci/examples.yml", "pipeline_slug": "rules-jvm-external-examples", "owned_by_bazel": True, }, - "rules_k8s": { - "git_repository": "https://github.com/bazelbuild/rules_k8s.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_k8s/master/.bazelci/presubmit.yml", - "pipeline_slug": "rules-k8s-k8s", - "disabled_reason": "https://github.com/bazelbuild/rules_k8s/issues/668", - }, "rules_kotlin": { "git_repository": "https://github.com/bazelbuild/rules_kotlin.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_kotlin/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-kotlin-kotlin", }, "rules_nodejs": { "git_repository": "https://github.com/bazelbuild/rules_nodejs.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_nodejs/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-nodejs-nodejs", }, "rules_perl": { "git_repository": "https://github.com/bazelbuild/rules_perl.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_perl/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-perl", }, "rules_proto": { "git_repository": "https://github.com/bazelbuild/rules_proto.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_proto/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-proto", "owned_by_bazel": True, }, "rules_python": { "git_repository": "https://github.com/bazelbuild/rules_python.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_python/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-python-python", "owned_by_bazel": True, }, "rules_rust": { "git_repository": "https://github.com/bazelbuild/rules_rust.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_rust/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-rust-rustlang", }, "rules_sass": { "git_repository": "https://github.com/bazelbuild/rules_sass.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_sass/main/.bazelci/presubmit.yml", "pipeline_slug": "rules-sass", }, "rules_scala": { "git_repository": "https://github.com/bazelbuild/rules_scala.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_scala/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-scala-scala", "disabled_reason": "waiting on https://github.com/bazelbuild/rules_scala/pull/1422", }, "rules_swift": { "git_repository": "https://github.com/bazelbuild/rules_swift.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_swift/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-swift-swift", }, "rules_testing": { "git_repository": "https://github.com/bazelbuild/rules_testing.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_testing/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-testing", "owned_by_bazel": True, }, "rules_webtesting": { "git_repository": "https://github.com/bazelbuild/rules_webtesting.git", - "http_config": "https://raw.githubusercontent.com/bazelbuild/rules_webtesting/master/.bazelci/presubmit.yml", "pipeline_slug": "rules-webtesting-saucelabs", }, "upb": { "git_repository": "https://github.com/protocolbuffers/upb.git", - "http_config": "https://raw.githubusercontent.com/protocolbuffers/upb/master/.bazelci/presubmit.yml", "pipeline_slug": "upb", "disabled_reason": "https://github.com/protocolbuffers/upb/issues/1290", }, @@ -455,7 +398,6 @@ "centos7": { "name": "CentOS 7 (OpenJDK 8, gcc 4.8.5)", "emoji-name": ":centos: 7 (OpenJDK 8, gcc 4.8.5)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7-java8", "python": "python3.6", @@ -463,7 +405,6 @@ "centos7_java11": { "name": "CentOS 7 (OpenJDK 11, gcc 4.8.5)", "emoji-name": ":centos: 7 (OpenJDK 11, gcc 4.8.5)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7-java11", "python": "python3.6", @@ -471,7 +412,6 @@ "centos7_java11_devtoolset10": { "name": "CentOS 7 (OpenJDK 11, gcc 10.2.1)", "emoji-name": ":centos: 7 (OpenJDK 11, gcc 10.2.1)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": ["ubuntu1404", "centos7", "linux"], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/centos7-java11-devtoolset10", "python": "python3.6", @@ -479,7 +419,6 @@ "debian10": { "name": "Debian 10 Buster (OpenJDK 11, gcc 8.3.0)", "emoji-name": ":debian: 10 Buster (OpenJDK 11, gcc 8.3.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/debian10-java11", "python": "python3.7", @@ -487,7 +426,6 @@ "debian11": { "name": "Debian 11 Bullseye (OpenJDK 17, gcc 10.2.1)", "emoji-name": ":debian: 11 Bullseye (OpenJDK 17, gcc 10.2.1)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/debian11-java17", "python": "python3.9", @@ -495,7 +433,6 @@ "ubuntu1604": { "name": "Ubuntu 16.04 LTS (OpenJDK 8, gcc 5.4.0)", "emoji-name": ":ubuntu: 16.04 LTS (OpenJDK 8, gcc 5.4.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1604-java8", "python": "python3.6", @@ -503,7 +440,6 @@ "ubuntu1804": { "name": "Ubuntu 18.04 LTS (OpenJDK 11, gcc 7.5.0)", "emoji-name": ":ubuntu: 18.04 LTS (OpenJDK 11, gcc 7.5.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu1804-java11", "python": "python3.6", @@ -511,7 +447,6 @@ "ubuntu2004": { "name": "Ubuntu 20.04 LTS (OpenJDK 11, gcc 9.4.0)", "emoji-name": ":ubuntu: 20.04 LTS (OpenJDK 11, gcc 9.4.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11", "python": "python3.8", @@ -519,7 +454,6 @@ "ubuntu2004_arm64": { "name": "Ubuntu 20.04 LTS ARM64 (OpenJDK 11, gcc 9.4.0)", "emoji-name": ":ubuntu: 20.04 LTS ARM64 (OpenJDK 11, gcc 9.4.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11", "python": "python3.8", @@ -530,7 +464,6 @@ "kythe_ubuntu2004": { "name": "Kythe (Ubuntu 20.04 LTS, OpenJDK 11, gcc 9.4.0)", "emoji-name": "Kythe (:ubuntu: 20.04 LTS, OpenJDK 11, gcc 9.4.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2004-java11-kythe", "python": "python3.8", @@ -538,7 +471,6 @@ "ubuntu2204": { "name": "Ubuntu 22.04 (OpenJDK 17, gcc 11.2.0)", "emoji-name": ":ubuntu: 22.04 (OpenJDK 17, gcc 11.2.0)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/ubuntu2204-java17", "python": "python3", @@ -546,7 +478,6 @@ "fedora39": { "name": "Fedora 39 (OpenJDK 17, gcc 13.1.1)", "emoji-name": ":fedora: 39 (OpenJDK 17, gcc 13.1.1)", - "downstream-root": "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "docker-image": f"gcr.io/{DOCKER_REGISTRY_PREFIX}/fedora39-java17", "python": "python3", @@ -554,7 +485,6 @@ "macos": { "name": "macOS (OpenJDK 11, Xcode)", "emoji-name": ":darwin: (OpenJDK 11, Xcode)", - "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": ["macos"], "queue": "macos", "python": "python3", @@ -562,7 +492,6 @@ "macos_qa": { "name": "macOS QA (OpenJDK 11, Xcode)", "emoji-name": ":darwin: :fire_extinguisher: (OpenJDK 11, Xcode)", - "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "queue": "macos_qa", "python": "python3", @@ -570,7 +499,6 @@ "macos_arm64_qa": { "name": "macOS arm64 QA (OpenJDK 8, Xcode)", "emoji-name": ":darwin: arm64 :fire_extinguisher: (OpenJDK 8, Xcode)", - "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": [], "queue": "macos_arm64_qa", "python": "python3", @@ -578,7 +506,6 @@ "macos_arm64": { "name": "macOS arm64 (OpenJDK 8, Xcode)", "emoji-name": ":darwin: arm64 (OpenJDK 8, Xcode)", - "downstream-root": "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": ["macos_arm64"], "queue": "macos_arm64", "python": "python3", @@ -586,7 +513,6 @@ "windows": { "name": "Windows (OpenJDK 11, VS2022)", "emoji-name": ":windows: (OpenJDK 11, VS2022)", - "downstream-root": "c:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": ["windows"], "queue": "windows", "python": "python.exe", @@ -594,7 +520,6 @@ "windows_arm64": { "name": "Windows ARM64 (OpenJDK 11, VS2022)", "emoji-name": ":windows: arm64 (OpenJDK 11, VS2022)", - "downstream-root": "c:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-downstream-projects", "publish_binary": ["windows_arm64"], # TODO(pcloudy): Switch to windows_arm64 queue when Windows ARM64 machines are available, # current we just use x86_64 machines to do cross compile. @@ -972,17 +897,6 @@ def gcloud_command(): return "gcloud.cmd" if is_windows() else "gcloud" -def downstream_projects_root(platform): - downstream_root = os.path.expandvars(PLATFORMS[platform]["downstream-root"]) - if platform == "windows" and os.path.exists("d:/b"): - # If this is a Windows machine with a local SSD, the build directory is - # on drive D. - downstream_root = downstream_root.replace("c:/b/", "d:/b/") - if not os.path.exists(downstream_root): - os.makedirs(downstream_root) - return downstream_root - - def match_matrix_attr_pattern(s): return re.match("^\${{\s*(\w+)\s*}}$", s) @@ -1229,9 +1143,6 @@ def calculate_flags(task_config, task_config_key, action_key, tmpdir, test_env_v def execute_commands( task_config, platform, - git_repository, - git_commit, - repo_location, use_bazel_at_commit, use_but, save_but, @@ -1290,11 +1201,6 @@ def execute_commands( os.environ["BAZELISK_USER_AGENT"] = "Bazelisk/BazelCI" test_env_vars.append("BAZELISK_USER_AGENT") - if repo_location: - os.chdir(repo_location) - elif git_repository: - clone_git_repository(git_repository, platform, git_commit) - # We use one binary for all Linux platforms (because we also just release one binary for all # Linux versions and we have to ensure that it works on all of them). binary_platform = platform if is_mac() or is_windows() else LINUX_BINARY_PLATFORM @@ -1378,9 +1284,8 @@ def PrepareRepoInCwd(print_cmd_groups, initial_setup=False): if needs_clean: execute_bazel_clean(bazel_binary, platform) - # The git_commit paramter only has a value in the downstream pipeline, - # but we need the commit here in order to calculate the correct targets. - git_commit = git_commit or os.getenv("BUILDKITE_COMMIT") + # we need the commit here in order to calculate the correct targets. + git_commit = os.getenv("BUILDKITE_COMMIT") if not git_commit: raise BuildkiteInfraException("Unable to determine Git commit for this build") @@ -1796,33 +1701,54 @@ def get_mirror_root(): return "/var/lib/bazelbuild/" +def get_repositories_root(): + """A root directory for preparing the reposioty to be tested.""" + repo_root = "/var/lib/buildkite-agent/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-org-repo-root" + if is_mac(): + repo_root = "/Users/buildkite/builds/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-org-repo-root" + elif is_windows(): + repo_root = "c:/b/${BUILDKITE_AGENT_NAME}/${BUILDKITE_ORGANIZATION_SLUG}-org-repo-root" + # If this is a Windows machine with a local SSD, the build directory is on drive D. + if os.path.exists("d:/b"): + repo_root = repo_root.replace("c:/b/", "d:/b/") + + repo_root = os.path.expandvars(repo_root) + + if not os.path.exists(repo_root): + os.makedirs(repo_root) + return repo_root + +def clone_git_repository(git_repository, git_commit=None, suppress_stdout=False): -def clone_git_repository(git_repository, platform, git_commit=None): - root = downstream_projects_root(platform) + def execute_git_command(args): + execute_command(args, print_output=not suppress_stdout, suppress_stdout=suppress_stdout) + + root = get_repositories_root() project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1) clone_path = os.path.join(root, project_name) - print_collapsed_group( - "Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD") - ) + if not suppress_stdout: + print_collapsed_group( + "Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD") + ) mirror_path = get_mirror_root() + re.sub(r"[^0-9A-Za-z]", "-", git_repository) if not os.path.exists(clone_path): if os.path.exists(mirror_path): - execute_command( + execute_git_command( ["git", "clone", "-v", "--reference", mirror_path, git_repository, clone_path] ) else: - execute_command(["git", "clone", "-v", git_repository, clone_path]) + execute_git_command(["git", "clone", "-v", git_repository, clone_path]) os.chdir(clone_path) - execute_command(["git", "remote", "set-url", "origin", git_repository]) - execute_command(["git", "clean", "-fdqx"]) - execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"]) - execute_command(["git", "fetch", "origin"]) + execute_git_command(["git", "remote", "set-url", "origin", git_repository]) + execute_git_command(["git", "clean", "-fdqx"]) + execute_git_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"]) + execute_git_command(["git", "fetch", "origin"]) if git_commit: # sync to a specific commit of this repository - execute_command(["git", "reset", git_commit, "--hard"]) + execute_git_command(["git", "reset", git_commit, "--hard"]) else: # sync to the latest commit of HEAD. Unlikely git pull this also works after a force push. remote_head = ( @@ -1830,12 +1756,12 @@ def clone_git_repository(git_repository, platform, git_commit=None): .decode("utf-8") .rstrip() ) - execute_command(["git", "reset", remote_head, "--hard"]) - execute_command(["git", "submodule", "sync", "--recursive"]) - execute_command(["git", "submodule", "update", "--init", "--recursive", "--force"]) - execute_command(["git", "submodule", "foreach", "--recursive", "git reset --hard"]) - execute_command(["git", "clean", "-fdqx"]) - execute_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"]) + execute_git_command(["git", "reset", remote_head, "--hard"]) + execute_git_command(["git", "submodule", "sync", "--recursive"]) + execute_git_command(["git", "submodule", "update", "--init", "--recursive", "--force"]) + execute_git_command(["git", "submodule", "foreach", "--recursive", "git reset --hard"]) + execute_git_command(["git", "clean", "-fdqx"]) + execute_git_command(["git", "submodule", "foreach", "--recursive", "git clean -fdqx"]) return clone_path @@ -2238,7 +2164,7 @@ def calculate_targets( shard_count = int(os.getenv("BUILDKITE_PARALLEL_JOB_COUNT", "-1")) sharding_enabled = shard_id > -1 and shard_count > -1 - use_bazel_diff = diffbase and can_use_bazel_diff() + use_bazel_diff = diffbase and can_use_bazel_diff(git_commit) # Skip target expansion if we don't need to calculate test targets if not use_bazel_diff and not sharding_enabled: @@ -2271,9 +2197,9 @@ def calculate_targets( return build_targets, actual_test_targets, coverage_targets, index_targets -def can_use_bazel_diff(): +def can_use_bazel_diff(git_commit): matched_files = [] - for f in get_modified_files(): + for f in get_modified_files(git_commit): for d in DISABLE_BAZEL_DIFF_IF_MODIFIED: if d.endswith("/") and f.startswith(d) or f == d: matched_files.append(f) @@ -2679,7 +2605,7 @@ def execute_command_and_get_output(args, shell=False, fail_if_nonzero=True, prin def execute_command( - args, shell=False, fail_if_nonzero=True, cwd=None, print_output=True, capture_stderr=False + args, shell=False, fail_if_nonzero=True, cwd=None, print_output=True, capture_stderr=False, suppress_stdout=False ): if print_output: eprint(" ".join(args)) @@ -2690,6 +2616,9 @@ def execute_command( env=os.environ, cwd=cwd, errors="replace", + stdout=subprocess.DEVNULL + if suppress_stdout + else subprocess.PIPE, # suppress_stdout=True when we don't want the output to be printed stderr=subprocess.PIPE if capture_stderr else None, # capture_stderr=True when we want exceptions to contain stderr @@ -2843,12 +2772,7 @@ def print_project_pipeline( ) # In Bazel Downstream Project pipelines, we should test the project at the last green commit. - git_commit = None - if is_downstream_pipeline(): - last_green_commit_url = bazelci_last_green_commit_url( - git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"] - ) - git_commit = get_last_green_commit(last_green_commit_url) + git_commit = get_last_green_commit(project_name) if is_downstream_pipeline() else None config_hashes = set() skipped_downstream_tasks = [] @@ -2960,7 +2884,7 @@ def print_project_pipeline( ) if "validate_config" in configs: - pipeline_steps += create_config_validation_steps() + pipeline_steps += create_config_validation_steps(git_commit or os.getenv("BUILDKITE_COMMIT")) if use_bazelisk_migrate() and not is_downstream_pipeline(): # Print results of bazelisk --migrate in project pipelines that explicitly set @@ -3022,10 +2946,10 @@ def get_platform_for_task(task, task_config): return task_config.get("platform", task) -def create_config_validation_steps(): +def create_config_validation_steps(git_commit): config_files = [ path - for path in get_modified_files() + for path in get_modified_files(git_commit) if path.startswith(".bazelci/") and os.path.splitext(path)[1] in CONFIG_FILE_EXTENSIONS ] return [ @@ -3043,9 +2967,9 @@ def create_config_validation_steps(): ] -def get_modified_files(): +def get_modified_files(git_commit): output = execute_command_and_get_output( - ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", os.getenv("BUILDKITE_COMMIT")] + ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", git_commit] ) return output.split("\n") @@ -3573,7 +3497,7 @@ def bazelci_last_green_downstream_commit_url(): return "gs://{}/last_green_commit/downstream_pipeline".format(bucket_name) -def get_last_green_commit(last_green_commit_url): +def get_last_green_commit_by_url(last_green_commit_url): try: return ( subprocess.check_output( @@ -3586,6 +3510,13 @@ def get_last_green_commit(last_green_commit_url): return None +def get_last_green_commit(project_name): + last_green_commit_url = bazelci_last_green_commit_url( + DOWNSTREAM_PROJECTS[project_name]["git_repository"], DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"] + ) + return get_last_green_commit_by_url(last_green_commit_url) + + def try_update_last_green_commit(): org_slug = os.getenv("BUILDKITE_ORGANIZATION_SLUG") pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG") @@ -3621,7 +3552,7 @@ def has_failed(job): def update_last_green_commit_if_newer(last_green_commit_url): - last_green_commit = get_last_green_commit(last_green_commit_url) + last_green_commit = get_last_green_commit_by_url(last_green_commit_url) current_commit = resolve_revision("HEAD") if last_green_commit: success = False @@ -3967,6 +3898,11 @@ def main(argv=None): notify=args.notify, ) elif args.subparsers_name == "project_pipeline": + # Fetch the repo in case we need to use file_config. + if args.git_repository: + git_commit = get_last_green_commit(args.project_name) if is_downstream_pipeline() else None + clone_git_repository(args.git_repository, git_commit, suppress_stdout=True) + configs = fetch_configs(args.http_config, args.file_config) print_project_pipeline( configs=configs, @@ -3979,6 +3915,12 @@ def main(argv=None): notify=args.notify, ) elif args.subparsers_name == "runner": + # Fetch the repo in case we need to use file_config. + if args.repo_location: + os.chdir(args.repo_location) + elif args.git_repository: + clone_git_repository(args.git_repository, args.git_commit) + configs = fetch_configs(args.http_config, args.file_config) tasks = configs.get("tasks", {}) task_config = tasks.get(args.task) @@ -4002,9 +3944,6 @@ def main(argv=None): execute_commands( task_config=task_config, platform=platform, - git_repository=args.git_repository, - git_commit=args.git_commit, - repo_location=args.repo_location, use_bazel_at_commit=args.use_bazel_at_commit, use_but=args.use_but, save_but=args.save_but, diff --git a/buildkite/culprit_finder.py b/buildkite/culprit_finder.py index c656fcbeb9..8001a17149 100755 --- a/buildkite/culprit_finder.py +++ b/buildkite/culprit_finder.py @@ -57,8 +57,9 @@ def get_bazel_commits_between(first_commit, second_commit): def get_configs(project_name): - http_config = bazelci.DOWNSTREAM_PROJECTS[project_name]["http_config"] - configs = bazelci.fetch_configs(http_config, None) + http_config = bazelci.DOWNSTREAM_PROJECTS[project_name].get("http_config") + file_config = bazelci.DOWNSTREAM_PROJECTS[project_name].get("file_config") + configs = bazelci.fetch_configs(http_config, file_config) return configs @@ -76,7 +77,8 @@ def get_tasks(project_name): def test_with_bazel_at_commit( project_name, task_name, repo_location, bazel_commit, needs_clean, repeat_times ): - http_config = bazelci.DOWNSTREAM_PROJECTS[project_name]["http_config"] + http_config = bazelci.DOWNSTREAM_PROJECTS[project_name].get("http_config") + file_config = bazelci.DOWNSTREAM_PROJECTS[project_name].get("file_config") for i in range(1, repeat_times + 1): if repeat_times > 1: bazelci.print_collapsed_group(":bazel: Try %s time" % i) @@ -85,10 +87,11 @@ def test_with_bazel_at_commit( [ "runner", "--task=" + task_name, - "--http_config=" + http_config, "--repo_location=" + repo_location, "--use_bazel_at_commit=" + bazel_commit, ] + + (["--http_config=" + http_config] if http_config else []) + + (["--file_config=" + file_config,] if file_config else []) + (["--needs_clean"] if needs_clean else []) ) except subprocess.CalledProcessError as e: @@ -99,14 +102,10 @@ def test_with_bazel_at_commit( return True -def clone_git_repository(project_name, task_name): - platform_name = get_platform(project_name, task_name) +def clone_git_repository(project_name, suppress_stdout=False): git_repository = bazelci.DOWNSTREAM_PROJECTS[project_name]["git_repository"] - last_green_commit_url = bazelci.bazelci_last_green_commit_url( - git_repository, bazelci.DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"] - ) - git_commit = bazelci.get_last_green_commit(last_green_commit_url) - return bazelci.clone_git_repository(git_repository, platform_name, git_commit) + git_commit = bazelci.get_last_green_commit(project_name) + return bazelci.clone_git_repository(git_repository, git_commit, suppress_stdout=suppress_stdout) def get_previous_bazel_commit(current_commit, count): @@ -229,6 +228,15 @@ def main(argv=None): try: project_name = os.environ["PROJECT_NAME"] + if project_name not in bazelci.DOWNSTREAM_PROJECTS: + raise Exception( + "Project name '%s' not recognized, available projects are %s" + % (project_name, str((bazelci.DOWNSTREAM_PROJECTS.keys()))) + ) + + # Clone the project repo so that we can get its CI config file at the same last green commit. + clone_git_repository(project_name, suppress_stdout=True) + # For old config file, we can still set PLATFORM_NAME as task name. task = os.environ.get("PLATFORM_NAME") or os.environ.get("TASK_NAME") if task: @@ -242,7 +250,7 @@ def main(argv=None): if not good_bazel_commit: # If GOOD_BAZEL_COMMIT is not set, use recorded last bazel green commit for downstream project last_green_commit_url = bazelci.bazelci_last_green_downstream_commit_url() - good_bazel_commit = bazelci.get_last_green_commit(last_green_commit_url) + good_bazel_commit = bazelci.get_last_green_commit_by_url(last_green_commit_url) bad_bazel_commit = os.environ.get("BAD_BAZEL_COMMIT") if not bad_bazel_commit: @@ -261,12 +269,6 @@ def main(argv=None): if "REPEAT_TIMES" in os.environ: repeat_times = int(os.environ["REPEAT_TIMES"]) - if project_name not in bazelci.DOWNSTREAM_PROJECTS: - raise Exception( - "Project name '%s' not recognized, available projects are %s" - % (project_name, str((bazelci.DOWNSTREAM_PROJECTS.keys()))) - ) - print_culprit_finder_pipeline( project_name=project_name, tasks=tasks, @@ -276,7 +278,7 @@ def main(argv=None): repeat_times=repeat_times, ) elif args.subparsers_name == "runner": - repo_location = clone_git_repository(args.project_name, args.task_name) + repo_location = clone_git_repository(args.project_name) good_bazel_commit, bad_bazel_commit = identify_bisect_range(args, repo_location) start_bisecting( project_name=args.project_name,