Allow downstream projects configs to be file_config (#1778)

Reapply #1776
with fixes:

- We need to properly clone the git repository so that we can get
`file_config` at the correct commit.
- Refactored `downstream-root` so that we don't need to get platform
info from task config for cloning the repo, which introduces a
chicken-egg problem.
- Allow suppressing stdout in `clone_git_repository` so that it won't
impact stdout of `project_pipeline`
- Make `validate_config` also work for downstream projects by adding
`git_commit` as a parameter of `get_modified_files`.

buildkite/bazel-bench/bazel_bench.py

@@ -77,7 +77,6 @@
 AGGR_JSON_PROFILES_FILENAME = "aggr_json_profiles.csv"
 PLATFORMS_WHITELIST = ['macos', 'ubuntu1804']
 REPORT_GENERATION_PLATFORM = 'ubuntu1804'
-STARTER_JOB_PLATFORM = 'ubuntu1804'
 
 
 def _bazel_bench_env_setup_command(platform, bazel_commits):
@@ -425,8 +424,7 @@ def main(args=None):
         else datetime.date.today()
     )
 
-    bazel_clone_path = bazelci.clone_git_repository(
-        BAZEL_REPOSITORY, STARTER_JOB_PLATFORM)
+    bazel_clone_path = bazelci.clone_git_repository(BAZEL_REPOSITORY)
     bazel_commits_full_list, bazel_commits_to_benchmark = _get_bazel_commits(
         date, bazel_clone_path, parsed_args.max_commits)
     bazel_bench_ci_steps = []

buildkite/bazel-central-registry/bcr_presubmit.py

@@ -154,16 +154,9 @@ def scratch_file(root, relative_path, lines=None, mode="w"):
     return abspath
 
 
-def get_root_dir(module_name, module_version, task, is_test_module=False):
-    # TODO(pcloudy): We use the "downstream root" as the repo root, find a better root path for BCR presubmit.
-    configs = get_test_module_task_config(module_name, module_version) if is_test_module else get_task_config(module_name, module_version)
-    platform = bazelci.get_platform_for_task(task, configs["tasks"][task])
-    return pathlib.Path(bazelci.downstream_projects_root(platform))
-
-
-def create_simple_repo(module_name, module_version, task):
+def create_simple_repo(module_name, module_version):
     """Create a simple Bazel module repo which depends on the target module."""
-    root = get_root_dir(module_name, module_version, task)
+    root = bazelci.get_repositories_root()
     scratch_file(root, "WORKSPACE")
     scratch_file(root, "BUILD")
     # TODO(pcloudy): Should we test this module as the root module? Maybe we do if we support dev dependency.
@@ -217,10 +210,10 @@ def unpack_archive(archive_file, output_dir):
     else:
         shutil.unpack_archive(archive_file, output_dir)
 
-def prepare_test_module_repo(module_name, module_version, task):
+def prepare_test_module_repo(module_name, module_version):
     """Prepare the test module repo and the presubmit yml file it should use"""
     bazelci.print_collapsed_group(":information_source: Prepare test module repo")
-    root = get_root_dir(module_name, module_version, task, is_test_module = True)
+    root = bazelci.get_repositories_root()
     source = load_source_json(module_name, module_version)
 
     # Download and unpack the source archive to ./output
@@ -420,11 +413,11 @@ def main(argv=None):
             pipeline_steps = [{"block": "Wait on BCR maintainer review", "blocked_state": "running"}] + pipeline_steps
         upload_jobs_to_pipeline(pipeline_steps)
     elif args.subparsers_name == "runner":
-        repo_location = create_simple_repo(args.module_name, args.module_version, args.task)
+        repo_location = create_simple_repo(args.module_name, args.module_version)
         config_file = get_presubmit_yml(args.module_name, args.module_version)
         return run_test(repo_location, config_file, args.task)
     elif args.subparsers_name == "test_module_runner":
-        repo_location, config_file = prepare_test_module_repo(args.module_name, args.module_version, args.task)
+        repo_location, config_file = prepare_test_module_repo(args.module_name, args.module_version)
         return run_test(repo_location, config_file, args.task)
     else:
         parser.print_help()

buildkite/bazel_auto_sheriff.py

@@ -126,11 +126,7 @@ def _get_main_build_result(self):
         job_infos = filter(lambda x: bool(x), (extract_job_info_by_key(job) for job in main_build_info["jobs"] if not ("soft_failed" in job and job["soft_failed"])))
         self.main_result["tasks"] = group_job_info_by_task(job_infos)
         self.main_result["state"] = get_project_state(self.main_result["tasks"])
-
-        last_green_commit_url = bazelci.bazelci_last_green_commit_url(
-            bazelci.DOWNSTREAM_PROJECTS[self.project]["git_repository"], self.pipeline
-        )
-        self.main_result["last_green_commit"] = bazelci.get_last_green_commit(last_green_commit_url)
+        self.main_result["last_green_commit"] = bazelci.get_last_green_commit(self.project)
 
 
     # Log all succeeded projects in the same annotate block