From 82de1d45506c5c895b30e2af72763eabdb1c6a95 Mon Sep 17 00:00:00 2001 From: anakinxc <103552181+anakinxc@users.noreply.github.com> Date: Wed, 3 Jan 2024 09:56:23 +0800 Subject: [PATCH] Use GCS for bazel cache (#456) --- .circleci/continue-config.yml | 35 +++++++++++---------------- libspu/mpc/cheetah/ot/emp/BUILD.bazel | 2 +- libspu/mpc/spdz2k/BUILD.bazel | 2 +- libspu/mpc/spdz2k/beaver/BUILD.bazel | 2 +- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml index daf3b370..a189a396 100644 --- a/.circleci/continue-config.yml +++ b/.circleci/continue-config.yml @@ -46,15 +46,12 @@ jobs: echo "Canceling workflow as too much time has elapsed" curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}" - checkout - - restore_cache: - name: "Restore build cache" - key: spu-build-comp-{{ arch }}- - run: name: Checkout devtools command: git clone https://github.com/secretflow/devtools.git ../devtools - run: - name: Decompress cache - command: sh ../devtools/decompress-build-cache.sh spu_build_cache + name: Setup GCS + command: ../devtools/base64_converter.py --str ${gcs_content} --out_file ../gcs.json - run: name: "Install dependencies" command: | @@ -63,30 +60,19 @@ jobs: python3 -m pip install -r requirements-dev.txt - run: name: "build" - command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --disk_cache=~/.cache/spu_build_cache --jobs 20 + command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --jobs 20 --remote_cache=https://storage.googleapis.com/secretflow --google_credentials=../gcs.json --remote_download_minimal - run: name: "test" command: | set +e declare -i test_status - bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --disk_cache=~/.cache/spu_build_cache | tee test_result.log; test_status=${PIPESTATUS[0]} + bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --remote_cache=https://storage.googleapis.com/secretflow --google_credentials=../gcs.json --remote_download_minimal | tee test_result.log; test_status=${PIPESTATUS[0]} sh ../devtools/rename-junit-xml.sh find bazel-bin/ -executable -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz exit ${test_status} - - run: - name: Cleanup and compress cache - command: | - sh ../devtools/clean_disk_cache.sh ~/.cache/spu_build_cache - sh ../devtools/compress-build-cache.sh spu_build_cache - when: always - - save_cache: - key: spu-build-comp-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }} - paths: - - /root/.cache/spu_build_cache.tar.gz - when: always - store_test_results: path: test-results - store_artifacts: @@ -114,15 +100,18 @@ jobs: conda init bash zsh pip install -r requirements.txt pip install -r requirements-dev.txt + - run: + name: Setup GCS + command: ../devtools/base64_converter.py --str ${gcs_content} --out_file ../gcs.json - run: name: "build" - command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning + command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --remote_cache=https://storage.googleapis.com/secretflow --google_credentials=../gcs.json --remote_download_minimal - run: name: "test" command: | set +e declare -i test_status - bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} + bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --remote_cache=https://storage.googleapis.com/secretflow --google_credentials=../gcs.json --remote_download_minimal | tee test_result.log; test_status=${PIPESTATUS[0]} sh ../devtools/rename-junit-xml.sh find bazel-bin/ -perm +111 -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz @@ -145,4 +134,8 @@ workflows: matrix: parameters: resource_class: ["2xlarge", "arm.2xlarge"] - - macOS_ut + context: + - gcs + - macOS_ut: + context: + - gcs diff --git a/libspu/mpc/cheetah/ot/emp/BUILD.bazel b/libspu/mpc/cheetah/ot/emp/BUILD.bazel index 9f4db581..0b024f08 100644 --- a/libspu/mpc/cheetah/ot/emp/BUILD.bazel +++ b/libspu/mpc/cheetah/ot/emp/BUILD.bazel @@ -40,10 +40,10 @@ spu_cc_library( spu_cc_test( name = "ferret_test", + timeout = "eternal", srcs = ["ferret_test.cc"], deps = [ ":ferret", "//libspu/mpc/utils:simulate", ], - timeout = "eternal", ) diff --git a/libspu/mpc/spdz2k/BUILD.bazel b/libspu/mpc/spdz2k/BUILD.bazel index 2e30d355..5e9e02b4 100644 --- a/libspu/mpc/spdz2k/BUILD.bazel +++ b/libspu/mpc/spdz2k/BUILD.bazel @@ -50,13 +50,13 @@ spu_cc_test( spu_cc_test( name = "protocol_ab_test", + timeout = "eternal", srcs = ["protocol_ab_test.cc"], deps = [ ":abprotocol_spdz2k_test", ":protocol", "//libspu/mpc:ab_api_test", ], - timeout = "eternal", ) spu_cc_library( diff --git a/libspu/mpc/spdz2k/beaver/BUILD.bazel b/libspu/mpc/spdz2k/beaver/BUILD.bazel index a7457384..03969e70 100644 --- a/libspu/mpc/spdz2k/beaver/BUILD.bazel +++ b/libspu/mpc/spdz2k/beaver/BUILD.bazel @@ -45,6 +45,7 @@ spu_cc_library( spu_cc_test( name = "beaver_test", + timeout = "eternal", srcs = ["beaver_test.cc"], deps = [ ":beaver_tfp", @@ -52,7 +53,6 @@ spu_cc_test( "//libspu/mpc/utils:simulate", "@com_google_googletest//:gtest", ], - timeout = "eternal", ) spu_cc_library(