From bd7886fe43d473549d691f2a8cd2ebd30d72c544 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Fri, 11 Oct 2024 13:49:37 +0400 Subject: [PATCH] test build on mac --- .github/workflows/build.yaml | 4 +- .../workflows/sanitizer-build-and-test.yaml | 494 +++++++++--------- .github/workflows/test.yaml | 460 ++++++++-------- 3 files changed, 479 insertions(+), 479 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 646e92f9..e95e7d51 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -62,12 +62,12 @@ jobs: submodules: "recursive" - name: Build id: build - run: sudo su -c "PG_VERSION=$PG_VERSION SETUP_ENV=1 USE_SOURCE=1 SETUP_POSTGRES=1 PACKAGE_EXTENSION=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh" + run: sudo sh -c "PG_VERSION=$PG_VERSION SETUP_ENV=1 USE_SOURCE=1 SETUP_POSTGRES=1 PACKAGE_EXTENSION=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh" env: PG_VERSION: ${{ matrix.postgres }} - name: Build Lantern CLI id: build_cli - run: sudo su -c "PACKAGE_CLI=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh" + run: sudo sh -c "PACKAGE_CLI=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh" if: ${{ matrix.postgres == 15 }} # run only once - name: Upload archive package artifacts if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }} diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index 9cd8c2cc..bd517515 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -1,247 +1,247 @@ -# Run tests with sanitizers enabled -# derived from from https://github.com/timescale/timescaledb/blob/main/.github/workflows/sanitizer-build-and-test.yaml -name: Sanitizer test -on: - push: - branches: - - main - - trigger/sanitizer - pull_request: - branches: - - main - release: - types: [created, edited] - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: "Run the build against llvm sanitizers" - required: false - default: false - -env: - name: "Sanitizer" - PG_SRC_DIR: "pgbuild" - PG_INSTALL_DIR: ${{ github.workspace }}/pgsql - extra_packages: "clang-15 llvm-15 llvm-15-dev llvm-15-tools" - llvm_config: "llvm-config-15" - CLANG: "clang-15" - CC: "clang-15" - CXX: "clang-15" - - CFLAGS: "-g -fsanitize=address -fno-omit-frame-pointer -Og -fno-inline-functions" - CXXFLAGS: "-g -fsanitize=address -fno-omit-frame-pointer -Og -fno-inline-functions" - LDFLAGS: "-fsanitize=address" - - ASAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_asan.txt - detect_odr_violation=0 log_path=${{ github.workspace }}/sanitizer/ - log_exe_name=true print_suppressions=false exitcode=27 - detect_leaks=0 abort_on_error=1 - - LSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_leak.txt - print_suppressions=0 log_path=${{ github.workspace }}/sanitizer/ - log_exe_name=true print_suppressions=false exitcode=27 - - UBSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_ub.txt - print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer/ - log_exe_name=true print_suppressions=false exitcode=27 - -jobs: - sanitizer: - # Change the JOB_NAME variable below when changing the name. - # Don't use the env variable here because the env context is not accessible. - name: PG${{ matrix.pg }} Sanitizer ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-22.04"] - pg: ["11.21", "12.16", "13.12", "14.9", "15.4", "16.0"] - steps: - - name: Enable UBSan if this is a release - if: ${{ github.event_name == 'release' }} - run: | - echo "CFLAGS=\"\-g -fsanitize=address,undefined -fno-omit-frame-pointer -O0 -fno-inline-functions"" >> $GITHUB_ENV - echo "CXXFLAGS=\"\-g -fsanitize=address,undefined -fno-omit-frame-pointer -O0 -fno-inline-functions"" >> $GITHUB_ENV - echo "LDFLAGS=\"-fsanitize=address,undefined\"" >> $GITHUB_ENV - - - name: Install Linux Dependencies - run: | - sudo apt-get update - sudo apt-get install -y wget \ - curl \ - systemd-coredump \ - build-essential \ - gdb \ - make \ - cmake \ - pkg-config \ - flex \ - bison \ - libicu-dev \ - libssl-dev \ - clang-15 \ - llvm-15 \ - llvm-15-dev \ - llvm-15-tools \ - libstdc++-12-dev \ - libstdc++6 - - - name: Checkout lantern - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: "recursive" - - # We are going to rebuild Postgres daily, so that it doesn't suddenly break - # ages after the original problem. - - name: Get date for build caching - id: get-date - run: | - echo "date=$(date +"%m-%y")" >> $GITHUB_OUTPUT - - # Create a directory for sanitizer logs. This directory is referenced by - # ASAN_OPTIONS, LSAN_OPTIONS, and UBSAN_OPTIONS - - name: Create sanitizer log directory - run: | - mkdir ${{ github.workspace }}/sanitizer - - # we cache the build directory instead of the install directory here - # because extension installation will write files to install directory - # leading to a tainted cache - - name: Cache PostgreSQL ${{ matrix.pg }} - id: cache-postgresql - uses: actions/cache@v4 - with: - path: ~/${{ env.PG_SRC_DIR }} - key: "${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}\ - -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - - - name: Build PostgreSQL ${{ matrix.pg }} if not in cache - id: build-postgresql - if: steps.cache-postgresql.outputs.cache-hit != 'true' - run: | - wget -q -O postgresql.tar.bz2 \ - https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2 - mkdir -p ~/$PG_SRC_DIR - tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 - # Add instrumentation to the Postgres memory contexts. For more details, see - # https://www.postgresql.org/message-id/CAM-w4HNH7%2BU9jZevpVK7Wr49tkfpWSR6wav0RLYrq0HWuP5cxw%40mail.gmail.com - patch -F5 -p1 -d ~/$PG_SRC_DIR < lantern_hnsw/scripts/sanitizers/postgres-asan-instrumentation.patch - cd ~/$PG_SRC_DIR - ./configure --prefix=$PG_INSTALL_DIR --enable-debug --enable-cassert \ - --with-openssl --without-readline --without-zlib --without-libxml - make -j$(nproc) - make -j$(nproc) -C contrib/pageinspect - make -j$(nproc) -C src/test/isolation - echo "exit_code=$?" >> $GITHUB_OUTPUT - - - name: save cache preemptively if postgres built - uses: actions/cache/save@v4 - if: steps.build-postgresql.outputs.exit_code == 0 - with: - path: ~/${{ env.PG_SRC_DIR }} - key: "${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}\ - -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - - - name: Upload config.log - if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' - uses: actions/upload-artifact@v4 - with: - name: config.log for PostgreSQL ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} - path: ~/${{ env.PG_SRC_DIR }}/config.log - - - name: Install PostgreSQL ${{ matrix.pg }} - run: | - make -C ~/$PG_SRC_DIR install - echo $PG_INSTALL_DIR/bin >> $GITHUB_PATH - sudo chmod -R 755 $PG_INSTALL_DIR/bin - $PG_INSTALL_DIR/bin/pg_config --version - - - name: Build Lantern - run: | - mkdir lantern_hnsw/lantern_build - cd lantern_hnsw/lantern_build - CXXFLAG="" cmake .. - make install - - - name: Start Postgres - run: | - mkdir -p $PG_INSTALL_DIR/data - #sudo chown -R postgres:postgres $PG_INSTALL_DIR/data - #sudo chown -R postgres:postgres ${{ github.workspace }}/sanitizer - #getent group postgres || sudo groupadd postgres - #id -u postgres &>/dev/null || sudo useradd -g postgres postgres - initdb -A trust -D $PG_INSTALL_DIR/data - postgres -D $PG_INSTALL_DIR/data >/tmp/postgres.log 2>&1 & - - - name: make test - run: | - cd lantern_hnsw/lantern_build - make test - - - name: Show regression diffs - if: always() - id: collectlogs - run: | - find /tmp/lantern -name regression.diffs -exec cat {} + > regression.log - cp /tmp/postgres.log . - if [[ "${{ runner.os }}" == "Linux" ]] ; then - # wait in case there are in-progress coredumps - sleep 10 - if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi - # print OOM killer information - sudo journalctl --system -q --facility=kern --grep "Killed process" || true - fi - if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi - #grep -e 'FAILED' -e 'failed (ignored)' installcheck.log || true - cat regression.log - - - name: Save regression diffs - if: always() && steps.collectlogs.outputs.regression_diff == 'true' - uses: actions/upload-artifact@v4 - with: - name: Regression diff ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: | - regression.log - - - name: Save postgres log - if: always() && steps.collectlogs.outputs.regression_diff == 'true' - uses: actions/upload-artifact@v4 - with: - name: Postgres log ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: | - postgres.log - - - name: Stack trace - if: always() && steps.collectlogs.outputs.coredumps == 'true' - run: | - sudo coredumpctl gdb <<<" - set verbose on - set trace-commands on - show debug-file-directory - printf "'"'"query = '%s'\n\n"'"'", debug_query_string - frame function ExceptionalCondition - printf "'"'"condition = '%s'\n"'"'", conditionName - up 1 - l - info args - info locals - bt full - " 2>&1 | tee stacktrace.log - ./lantern_hnsw/scripts/sanitizers/bundle_coredump.sh - grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: - - - name: Coredumps - if: always() && steps.collectlogs.outputs.coredumps == 'true' - uses: actions/upload-artifact@v4 - with: - name: Coredumps ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: coredumps - - - name: sanitizer logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: ${{ github.workspace }}/sanitizer +# # Run tests with sanitizers enabled +# # derived from from https://github.com/timescale/timescaledb/blob/main/.github/workflows/sanitizer-build-and-test.yaml +# name: Sanitizer test +# on: +# push: +# branches: +# - main +# - trigger/sanitizer +# pull_request: +# branches: +# - main +# release: +# types: [created, edited] +# workflow_dispatch: +# inputs: +# debug_enabled: +# type: boolean +# description: "Run the build against llvm sanitizers" +# required: false +# default: false +# +# env: +# name: "Sanitizer" +# PG_SRC_DIR: "pgbuild" +# PG_INSTALL_DIR: ${{ github.workspace }}/pgsql +# extra_packages: "clang-15 llvm-15 llvm-15-dev llvm-15-tools" +# llvm_config: "llvm-config-15" +# CLANG: "clang-15" +# CC: "clang-15" +# CXX: "clang-15" +# +# CFLAGS: "-g -fsanitize=address -fno-omit-frame-pointer -Og -fno-inline-functions" +# CXXFLAGS: "-g -fsanitize=address -fno-omit-frame-pointer -Og -fno-inline-functions" +# LDFLAGS: "-fsanitize=address" +# +# ASAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_asan.txt +# detect_odr_violation=0 log_path=${{ github.workspace }}/sanitizer/ +# log_exe_name=true print_suppressions=false exitcode=27 +# detect_leaks=0 abort_on_error=1 +# +# LSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_leak.txt +# print_suppressions=0 log_path=${{ github.workspace }}/sanitizer/ +# log_exe_name=true print_suppressions=false exitcode=27 +# +# UBSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_ub.txt +# print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer/ +# log_exe_name=true print_suppressions=false exitcode=27 +# +# jobs: +# sanitizer: +# # Change the JOB_NAME variable below when changing the name. +# # Don't use the env variable here because the env context is not accessible. +# name: PG${{ matrix.pg }} Sanitizer ${{ matrix.os }} +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: ["ubuntu-22.04"] +# pg: ["11.21", "12.16", "13.12", "14.9", "15.4", "16.0"] +# steps: +# - name: Enable UBSan if this is a release +# if: ${{ github.event_name == 'release' }} +# run: | +# echo "CFLAGS=\"\-g -fsanitize=address,undefined -fno-omit-frame-pointer -O0 -fno-inline-functions"" >> $GITHUB_ENV +# echo "CXXFLAGS=\"\-g -fsanitize=address,undefined -fno-omit-frame-pointer -O0 -fno-inline-functions"" >> $GITHUB_ENV +# echo "LDFLAGS=\"-fsanitize=address,undefined\"" >> $GITHUB_ENV +# +# - name: Install Linux Dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y wget \ +# curl \ +# systemd-coredump \ +# build-essential \ +# gdb \ +# make \ +# cmake \ +# pkg-config \ +# flex \ +# bison \ +# libicu-dev \ +# libssl-dev \ +# clang-15 \ +# llvm-15 \ +# llvm-15-dev \ +# llvm-15-tools \ +# libstdc++-12-dev \ +# libstdc++6 +# +# - name: Checkout lantern +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# submodules: "recursive" +# +# # We are going to rebuild Postgres daily, so that it doesn't suddenly break +# # ages after the original problem. +# - name: Get date for build caching +# id: get-date +# run: | +# echo "date=$(date +"%m-%y")" >> $GITHUB_OUTPUT +# +# # Create a directory for sanitizer logs. This directory is referenced by +# # ASAN_OPTIONS, LSAN_OPTIONS, and UBSAN_OPTIONS +# - name: Create sanitizer log directory +# run: | +# mkdir ${{ github.workspace }}/sanitizer +# +# # we cache the build directory instead of the install directory here +# # because extension installation will write files to install directory +# # leading to a tainted cache +# - name: Cache PostgreSQL ${{ matrix.pg }} +# id: cache-postgresql +# uses: actions/cache@v4 +# with: +# path: ~/${{ env.PG_SRC_DIR }} +# key: "${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}\ +# -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" +# +# - name: Build PostgreSQL ${{ matrix.pg }} if not in cache +# id: build-postgresql +# if: steps.cache-postgresql.outputs.cache-hit != 'true' +# run: | +# wget -q -O postgresql.tar.bz2 \ +# https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2 +# mkdir -p ~/$PG_SRC_DIR +# tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 +# # Add instrumentation to the Postgres memory contexts. For more details, see +# # https://www.postgresql.org/message-id/CAM-w4HNH7%2BU9jZevpVK7Wr49tkfpWSR6wav0RLYrq0HWuP5cxw%40mail.gmail.com +# patch -F5 -p1 -d ~/$PG_SRC_DIR < lantern_hnsw/scripts/sanitizers/postgres-asan-instrumentation.patch +# cd ~/$PG_SRC_DIR +# ./configure --prefix=$PG_INSTALL_DIR --enable-debug --enable-cassert \ +# --with-openssl --without-readline --without-zlib --without-libxml +# make -j$(nproc) +# make -j$(nproc) -C contrib/pageinspect +# make -j$(nproc) -C src/test/isolation +# echo "exit_code=$?" >> $GITHUB_OUTPUT +# +# - name: save cache preemptively if postgres built +# uses: actions/cache/save@v4 +# if: steps.build-postgresql.outputs.exit_code == 0 +# with: +# path: ~/${{ env.PG_SRC_DIR }} +# key: "${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}\ +# -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" +# +# - name: Upload config.log +# if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: config.log for PostgreSQL ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} +# path: ~/${{ env.PG_SRC_DIR }}/config.log +# +# - name: Install PostgreSQL ${{ matrix.pg }} +# run: | +# make -C ~/$PG_SRC_DIR install +# echo $PG_INSTALL_DIR/bin >> $GITHUB_PATH +# sudo chmod -R 755 $PG_INSTALL_DIR/bin +# $PG_INSTALL_DIR/bin/pg_config --version +# +# - name: Build Lantern +# run: | +# mkdir lantern_hnsw/lantern_build +# cd lantern_hnsw/lantern_build +# CXXFLAG="" cmake .. +# make install +# +# - name: Start Postgres +# run: | +# mkdir -p $PG_INSTALL_DIR/data +# #sudo chown -R postgres:postgres $PG_INSTALL_DIR/data +# #sudo chown -R postgres:postgres ${{ github.workspace }}/sanitizer +# #getent group postgres || sudo groupadd postgres +# #id -u postgres &>/dev/null || sudo useradd -g postgres postgres +# initdb -A trust -D $PG_INSTALL_DIR/data +# postgres -D $PG_INSTALL_DIR/data >/tmp/postgres.log 2>&1 & +# +# - name: make test +# run: | +# cd lantern_hnsw/lantern_build +# make test +# +# - name: Show regression diffs +# if: always() +# id: collectlogs +# run: | +# find /tmp/lantern -name regression.diffs -exec cat {} + > regression.log +# cp /tmp/postgres.log . +# if [[ "${{ runner.os }}" == "Linux" ]] ; then +# # wait in case there are in-progress coredumps +# sleep 10 +# if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi +# # print OOM killer information +# sudo journalctl --system -q --facility=kern --grep "Killed process" || true +# fi +# if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi +# #grep -e 'FAILED' -e 'failed (ignored)' installcheck.log || true +# cat regression.log +# +# - name: Save regression diffs +# if: always() && steps.collectlogs.outputs.regression_diff == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: Regression diff ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} +# path: | +# regression.log +# +# - name: Save postgres log +# if: always() && steps.collectlogs.outputs.regression_diff == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: Postgres log ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} +# path: | +# postgres.log +# +# - name: Stack trace +# if: always() && steps.collectlogs.outputs.coredumps == 'true' +# run: | +# sudo coredumpctl gdb <<<" +# set verbose on +# set trace-commands on +# show debug-file-directory +# printf "'"'"query = '%s'\n\n"'"'", debug_query_string +# frame function ExceptionalCondition +# printf "'"'"condition = '%s'\n"'"'", conditionName +# up 1 +# l +# info args +# info locals +# bt full +# " 2>&1 | tee stacktrace.log +# ./lantern_hnsw/scripts/sanitizers/bundle_coredump.sh +# grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: +# +# - name: Coredumps +# if: always() && steps.collectlogs.outputs.coredumps == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: Coredumps ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} +# path: coredumps +# +# - name: sanitizer logs +# if: always() +# uses: actions/upload-artifact@v4 +# with: +# name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} +# path: ${{ github.workspace }}/sanitizer diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 713390d1..dbb414c5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,230 +1,230 @@ -name: test -on: - push: - branches: - - main - - dev - pull_request: - branches: - - main - - dev - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" - required: false - default: false -jobs: - test_lantern: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04, ubicloud-standard-4-arm, macos-13] - postgres: [11, 12, 13, 14, 15, 16] - steps: - - uses: actions/checkout@v4 - with: - # fetch-depth ensures all tags are present on the repo so we can run update tests successfully - fetch-depth: 0 - submodules: "recursive" - - - name: Build without SSL - id: build_without_ssl - run: sudo sh -c "INSTALL_EXTRAS=1 USE_SSL=0 PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT INSTALL_CLI=1 ENABLE_FAILURE_POINTS=1 ./ci/scripts/build.sh" - env: - PG_VERSION: ${{ matrix.postgres }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - - name: Check clang formatting - run: "cd /tmp/lantern/lantern_hnsw/build && make format_check" - # Enable tmate debugging of manually-triggered workflows if the input option was provided - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Run tests linux without SSL - id: test-linux-without-ssl - run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ !startsWith(matrix.os, 'mac') }} - - name: Run integration tests linux without SSL - id: integration-test-linux-without-ssl - run: | - # pytest tries to open files with full path and so 'work' home folder must be listable by postgres for it to work - sudo -E chmod +x $HOME && \ - sudo su postgres -c "LANTERN_CLI_PATH=${{ env.LANTERN_CLI_PATH }} USE_SSL=0 /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py -k external_index" &&\ - echo "Done with integration tests" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ !startsWith(matrix.os, 'mac') }} - - - name: Build - id: build - run: sudo sh -c "USE_SSL=1 PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=$ENABLE_COVERAGE ENABLE_FAILURE_POINTS=1 ./ci/scripts/build.sh" - env: - PG_VERSION: ${{ matrix.postgres }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - ENABLE_COVERAGE: ${{ (startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15) && '1' || '0' }} - - name: Run tests linux - id: test-linux - run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh" - env: - PG_VERSION: ${{ matrix.postgres }} - ENABLE_COVERAGE: ${{ (startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15) && '1' || '0' }} - if: ${{ !startsWith(matrix.os, 'mac') }} - # integration tests - - name: Set LANTERN_CLI_PATH variable - run: echo "LANTERN_CLI_PATH=/tmp/bin/lantern-cli" >> $GITHUB_ENV - if: ${{ !startsWith(matrix.os, 'mac') }} - - name: Run integration tests linux - id: integration-test-linux - run: | - # pytest tries to open files with full path and so 'work' home folder must be listable by postgres for it to work - sudo fuser -k 8998/tcp || true 2>/dev/null # kill previously started cli - sudo -E chmod +x $HOME && \ - sudo su postgres -c "LANTERN_CLI_PATH=${{ env.LANTERN_CLI_PATH }} USE_SSL=1 /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py" &&\ - echo "Done with integration tests" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ !startsWith(matrix.os, 'mac') }} - - - name: Collect coverage files - id: collect-cov-files - run: sudo su postgres -c "cd /tmp/lantern/lantern_hnsw/build && make cover" - if: ${{ startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} - - - name: Run update tests linux - id: update-test-linux - run: | - # Start postgres - sudo su postgres -c "PG_VERSION=$PG_VERSION RUN_TESTS=0 ./ci/scripts/run-tests-linux.sh" && \ - sudo su -c "cd /tmp/lantern/lantern_hnsw && PG_VERSION=$PG_VERSION ./cienv/bin/python ./scripts/test_updates.py -U postgres --builddir ./build_updates --rootdir ." &&\ - echo "Done with updates" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ !startsWith(matrix.os, 'mac') }} - - name: Run catalog update checks - id: catalog-update-test-linux - run: | - sudo su postgres -c "PG_VERSION=$PG_VERSION RUN_TESTS=0 ./ci/scripts/run-tests-linux.sh" && \ - sudo su -c "cd /tmp/lantern/lantern_hnsw && PG_VERSION=$PG_VERSION DATABASE_URL='postgresql://postgres@localhost:5432/postgres' ruby ./scripts/test_updates/main.rb test lantern" &&\ - echo "Done catalog update checks" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ !startsWith(matrix.os, 'mac') }} - - - name: Run tests mac - id: test-mac - run: ./ci/scripts/run-tests-mac.sh - env: - PG_VERSION: ${{ matrix.postgres }} - # postgresql@11 seems broken on brew on Ventura used in the runner https://github.com/orgs/Homebrew/discussions/5263 - if: ${{ startsWith(matrix.os, 'mac') && matrix.postgres != 11 }} - - name: Run integration tests mac - id: integration-test-mac - run: | - PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py &&\ - echo "Done with integration tests" - env: - PG_VERSION: ${{ matrix.postgres }} - if: ${{ startsWith(matrix.os, 'mac') && matrix.postgres != 11 }} - - name: Upload Postgres logs - if: failure() - uses: actions/upload-artifact@v4 - with: - name: postgres-server-${{ matrix.os }}-${{ matrix.postgres }}-logs - path: | - /tmp/pg-out.log - /tmp/pg-error.log - - name: Upload lantern coverage - if: ${{ startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} - uses: actions/upload-artifact@v4 - with: - name: lantern-lcov.info - path: /tmp/lantern/lantern_hnsw/build/coverage.info - test_extras: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04, ubicloud-standard-4-arm] - postgres: [15] - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - name: Setup Rust - run: rustup toolchain install stable --profile minimal --no-self-update - - name: Configure sccache - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.4 - - name: Cache cargo deps - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - lantern_cli - lantern_extras - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: Setup test env - run: sudo su -c "PG_VERSION=$PG_VERSION SETUP_ENV=1 SETUP_POSTGRES=1 SETUP_TESTS=1 USE_SOURCE=1 ./ci/scripts/build-extras.sh" - env: - PG_VERSION: ${{ matrix.postgres }} - - name: Install extension - run: | - cargo install cargo-pgrx --version 0.11.3 - cargo pgrx init "--pg$PG_VERSION" /usr/bin/pg_config - RUSTFLAGS="--cfg profile=\"ci-build\"" cargo pgrx install --sudo --pg-config /usr/bin/pg_config --package lantern_extras - env: - PG_VERSION: ${{ matrix.postgres }} - - name: Run postgres - run: RUN_POSTGRES=1 ./ci/scripts/build-extras.sh - env: - PG_VERSION: ${{ matrix.postgres }} - - name: Setup permissions - run: | - sudo chmod 777 -R /usr/lib/postgresql/15/lib/ - sudo chmod 777 -R /usr/share/postgresql/15/extension/ - - name: Run tests - run: cargo llvm-cov --workspace --lcov --output-path lantern-extras-lcov.info - env: - OPENAI_TOKEN: ${{ secrets.OPENAI_TOKEN }} - COHERE_TOKEN: ${{ secrets.COHERE_TOKEN }} - DB_URL: "postgres://postgres@127.0.0.1:5432/postgres" - - name: Upload lantern_extras coverage - uses: actions/upload-artifact@v4 - if: ${{ startsWith(matrix.os, 'ubuntu') }} - with: - name: lantern-extras-lcov.info - path: ./lantern-extras-lcov.info - upload_coverage: - runs-on: ubuntu-22.04 - needs: [test_lantern, test_extras] - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - uses: actions/download-artifact@v4 - with: - pattern: "*-lcov.info" - merge-multiple: true - path: /tmp/lantern-coverage - - uses: geekyeggo/delete-artifact@v4 - with: - name: "*-lcov.info" - - name: Upload coverage to Codecov - env: - codecov_token: ${{ secrets.CODECOV_TOKEN }} - uses: codecov/codecov-action@v4.0.1 - if: ${{ env.codecov_token != '' }} - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: /tmp/lantern-coverage/coverage.info,/tmp/lantern-coverage/lantern-extras-lcov.info - fail_ci_if_error: true - verbose: true +# name: test +# on: +# push: +# branches: +# - main +# - dev +# pull_request: +# branches: +# - main +# - dev +# workflow_dispatch: +# inputs: +# debug_enabled: +# type: boolean +# description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" +# required: false +# default: false +# jobs: +# test_lantern: +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: [ubuntu-22.04, ubicloud-standard-4-arm, macos-13] +# postgres: [11, 12, 13, 14, 15, 16] +# steps: +# - uses: actions/checkout@v4 +# with: +# # fetch-depth ensures all tags are present on the repo so we can run update tests successfully +# fetch-depth: 0 +# submodules: "recursive" +# +# - name: Build without SSL +# id: build_without_ssl +# run: sudo sh -c "INSTALL_EXTRAS=1 USE_SSL=0 PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT INSTALL_CLI=1 ENABLE_FAILURE_POINTS=1 ./ci/scripts/build.sh" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +# - name: Check clang formatting +# run: "cd /tmp/lantern/lantern_hnsw/build && make format_check" +# # Enable tmate debugging of manually-triggered workflows if the input option was provided +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 +# if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} +# - name: Run tests linux without SSL +# id: test-linux-without-ssl +# run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# - name: Run integration tests linux without SSL +# id: integration-test-linux-without-ssl +# run: | +# # pytest tries to open files with full path and so 'work' home folder must be listable by postgres for it to work +# sudo -E chmod +x $HOME && \ +# sudo su postgres -c "LANTERN_CLI_PATH=${{ env.LANTERN_CLI_PATH }} USE_SSL=0 /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py -k external_index" &&\ +# echo "Done with integration tests" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# +# - name: Build +# id: build +# run: sudo sh -c "USE_SSL=1 PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=$ENABLE_COVERAGE ENABLE_FAILURE_POINTS=1 ./ci/scripts/build.sh" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +# ENABLE_COVERAGE: ${{ (startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15) && '1' || '0' }} +# - name: Run tests linux +# id: test-linux +# run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# ENABLE_COVERAGE: ${{ (startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15) && '1' || '0' }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# # integration tests +# - name: Set LANTERN_CLI_PATH variable +# run: echo "LANTERN_CLI_PATH=/tmp/bin/lantern-cli" >> $GITHUB_ENV +# if: ${{ !startsWith(matrix.os, 'mac') }} +# - name: Run integration tests linux +# id: integration-test-linux +# run: | +# # pytest tries to open files with full path and so 'work' home folder must be listable by postgres for it to work +# sudo fuser -k 8998/tcp || true 2>/dev/null # kill previously started cli +# sudo -E chmod +x $HOME && \ +# sudo su postgres -c "LANTERN_CLI_PATH=${{ env.LANTERN_CLI_PATH }} USE_SSL=1 /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py" &&\ +# echo "Done with integration tests" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# +# - name: Collect coverage files +# id: collect-cov-files +# run: sudo su postgres -c "cd /tmp/lantern/lantern_hnsw/build && make cover" +# if: ${{ startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} +# +# - name: Run update tests linux +# id: update-test-linux +# run: | +# # Start postgres +# sudo su postgres -c "PG_VERSION=$PG_VERSION RUN_TESTS=0 ./ci/scripts/run-tests-linux.sh" && \ +# sudo su -c "cd /tmp/lantern/lantern_hnsw && PG_VERSION=$PG_VERSION ./cienv/bin/python ./scripts/test_updates.py -U postgres --builddir ./build_updates --rootdir ." &&\ +# echo "Done with updates" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# - name: Run catalog update checks +# id: catalog-update-test-linux +# run: | +# sudo su postgres -c "PG_VERSION=$PG_VERSION RUN_TESTS=0 ./ci/scripts/run-tests-linux.sh" && \ +# sudo su -c "cd /tmp/lantern/lantern_hnsw && PG_VERSION=$PG_VERSION DATABASE_URL='postgresql://postgres@localhost:5432/postgres' ruby ./scripts/test_updates/main.rb test lantern" &&\ +# echo "Done catalog update checks" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ !startsWith(matrix.os, 'mac') }} +# +# - name: Run tests mac +# id: test-mac +# run: ./ci/scripts/run-tests-mac.sh +# env: +# PG_VERSION: ${{ matrix.postgres }} +# # postgresql@11 seems broken on brew on Ventura used in the runner https://github.com/orgs/Homebrew/discussions/5263 +# if: ${{ startsWith(matrix.os, 'mac') && matrix.postgres != 11 }} +# - name: Run integration tests mac +# id: integration-test-mac +# run: | +# PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" /tmp/lantern/lantern_hnsw/cienv/bin/python ./lantern_hnsw/scripts/integration_tests.py &&\ +# echo "Done with integration tests" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# if: ${{ startsWith(matrix.os, 'mac') && matrix.postgres != 11 }} +# - name: Upload Postgres logs +# if: failure() +# uses: actions/upload-artifact@v4 +# with: +# name: postgres-server-${{ matrix.os }}-${{ matrix.postgres }}-logs +# path: | +# /tmp/pg-out.log +# /tmp/pg-error.log +# - name: Upload lantern coverage +# if: ${{ startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} +# uses: actions/upload-artifact@v4 +# with: +# name: lantern-lcov.info +# path: /tmp/lantern/lantern_hnsw/build/coverage.info +# test_extras: +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: [ubuntu-22.04, ubicloud-standard-4-arm] +# postgres: [15] +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: "recursive" +# - name: Setup Rust +# run: rustup toolchain install stable --profile minimal --no-self-update +# - name: Configure sccache +# run: | +# echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV +# echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV +# - name: Run sccache-cache +# uses: mozilla-actions/sccache-action@v0.0.4 +# - name: Cache cargo deps +# uses: Swatinem/rust-cache@v2 +# with: +# workspaces: | +# lantern_cli +# lantern_extras +# - name: Install cargo-llvm-cov +# uses: taiki-e/install-action@cargo-llvm-cov +# - name: Setup test env +# run: sudo su -c "PG_VERSION=$PG_VERSION SETUP_ENV=1 SETUP_POSTGRES=1 SETUP_TESTS=1 USE_SOURCE=1 ./ci/scripts/build-extras.sh" +# env: +# PG_VERSION: ${{ matrix.postgres }} +# - name: Install extension +# run: | +# cargo install cargo-pgrx --version 0.11.3 +# cargo pgrx init "--pg$PG_VERSION" /usr/bin/pg_config +# RUSTFLAGS="--cfg profile=\"ci-build\"" cargo pgrx install --sudo --pg-config /usr/bin/pg_config --package lantern_extras +# env: +# PG_VERSION: ${{ matrix.postgres }} +# - name: Run postgres +# run: RUN_POSTGRES=1 ./ci/scripts/build-extras.sh +# env: +# PG_VERSION: ${{ matrix.postgres }} +# - name: Setup permissions +# run: | +# sudo chmod 777 -R /usr/lib/postgresql/15/lib/ +# sudo chmod 777 -R /usr/share/postgresql/15/extension/ +# - name: Run tests +# run: cargo llvm-cov --workspace --lcov --output-path lantern-extras-lcov.info +# env: +# OPENAI_TOKEN: ${{ secrets.OPENAI_TOKEN }} +# COHERE_TOKEN: ${{ secrets.COHERE_TOKEN }} +# DB_URL: "postgres://postgres@127.0.0.1:5432/postgres" +# - name: Upload lantern_extras coverage +# uses: actions/upload-artifact@v4 +# if: ${{ startsWith(matrix.os, 'ubuntu') }} +# with: +# name: lantern-extras-lcov.info +# path: ./lantern-extras-lcov.info +# upload_coverage: +# runs-on: ubuntu-22.04 +# needs: [test_lantern, test_extras] +# strategy: +# fail-fast: false +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: "recursive" +# - uses: actions/download-artifact@v4 +# with: +# pattern: "*-lcov.info" +# merge-multiple: true +# path: /tmp/lantern-coverage +# - uses: geekyeggo/delete-artifact@v4 +# with: +# name: "*-lcov.info" +# - name: Upload coverage to Codecov +# env: +# codecov_token: ${{ secrets.CODECOV_TOKEN }} +# uses: codecov/codecov-action@v4.0.1 +# if: ${{ env.codecov_token != '' }} +# with: +# token: ${{ secrets.CODECOV_TOKEN }} +# files: /tmp/lantern-coverage/coverage.info,/tmp/lantern-coverage/lantern-extras-lcov.info +# fail_ci_if_error: true +# verbose: true