From f512e1b2c6c1c777cf96afac013eb5cf54b3fa8f Mon Sep 17 00:00:00 2001 From: Martin Studer Date: Mon, 28 Oct 2024 10:11:02 +0100 Subject: [PATCH] test --- .Rbuildignore | 1 + .github/workflows/ci-tests.yml | 4 +- spark_versions.json | 326 +++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 spark_versions.json diff --git a/.Rbuildignore b/.Rbuildignore index 001dd1a..49a77d8 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -13,3 +13,4 @@ cran-comments.md ^\.github$ ^adc.json$ +^spark_versions.json$ \ No newline at end of file diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2a7ff86..b0288c4 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -25,11 +25,10 @@ jobs: fail-fast: false matrix: config: - # Spark 3.5 seems currently unavailable so we use 3.4 for now - os-name: ubuntu os-version: latest java: 17 - spark: "3.4" + spark: "3.5" r-version: release # - os-name: ubuntu # os-version: latest @@ -67,6 +66,7 @@ jobs: BIGQUERY_MATERIALIZATION_DATASET: test BIGQUERY_APPLICATION_CREDENTIALS: ${{ github.workspace }}/adc.json R_DEFAULT_INTERNET_TIMEOUT: 1800 + R_SPARKINSTALL_INSTALL_INFO_PATH: ${{ github.workspace }}/spark_versions.json steps: - uses: actions/checkout@v4 diff --git a/spark_versions.json b/spark_versions.json new file mode 100644 index 0000000..18f41d5 --- /dev/null +++ b/spark_versions.json @@ -0,0 +1,326 @@ +[ + { + "spark": "2.4.0", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.0", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.1", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.1", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.2", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.2", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.3", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.3", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.4", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.4", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.5", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.5/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.5", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.5/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.6", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.6/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.6", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.6/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.7", + "hadoop": "2.6", + "base": "https://archive.apache.org/dist/spark/spark-2.4.7/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "2.4.7", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-2.4.7/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.0", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.0.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.0", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.0.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.1", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.0.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.1", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.0.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.2", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.0.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.2", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.0.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.3", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.0.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.0.3", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.0.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.1", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.1.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.1", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.1.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.2", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.1.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.2", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.1.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.3", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.1.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.1.3", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.1.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.0", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.2.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.0", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.2.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.1", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.2.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.1", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.2.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.2", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.2.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.2", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.2.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.3", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.2.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.3", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.2.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.4", + "hadoop": "2.7", + "base": "https://archive.apache.org/dist/spark/spark-3.2.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.2.4", + "hadoop": "3.2", + "base": "https://archive.apache.org/dist/spark/spark-3.2.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.3.0", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.3.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.3.1", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.3.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.3.2", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.3.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.3.3", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.3.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.3.4", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.3.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.4.0", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.4.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.4.1", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.4.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.4.2", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.4.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.4.3", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.4.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.4.4", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.4.4/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.5.0", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.5.0/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.5.1", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.5.1/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.5.2", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.5.2/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + }, + { + "spark": "3.5.3", + "hadoop": "3", + "base": "https://archive.apache.org/dist/spark/spark-3.5.3/", + "pattern": "spark-%s-bin-hadoop%s.tgz" + } +] \ No newline at end of file