Move lantern_extras to lantern repo

.cargo/config

@@ -0,0 +1,10 @@
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-args=-Wl,-rpath,/usr/local/lib/onnxruntime/lib"]
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-args=-Wl,-rpath,/usr/local/lib/onnxruntime/lib"]
+[target.'cfg(target_os="macos")']
+# Postgres symbols won't be available until runtime
+rustflags = ["-Clink-arg=-Wl,-undefined,dynamic_lookup"]
+[env]
+ORT_STRATEGY = { value = "system" }
+ORT_LIB_LOCATION = { value = "/usr/local/lib/onnxruntime/lib" , relative = false }

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.cargo
+.github
+target
+.DS_Store
+lantern_hnsw/build

.github/workflows/build.yaml

@@ -16,12 +16,12 @@ on:
         required: false
         default: false
 jobs:
-  build:
+  build_lantern:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubicloud-standard-2-arm, macos-13]
+        os: [ubuntu-22.04, ubicloud-standard-4-arm, macos-13]
         postgres: [11, 12, 13, 14, 15, 16]
     steps:
       - uses: actions/checkout@v4
@@ -37,23 +37,69 @@ jobs:
         run: sudo sh -c "GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/package-archive.sh"
       - name: Upload deb package artifacts
         uses: actions/upload-artifact@v4
-        if: ${{ steps.build.outputs.deb_package_path != '' }}
+        if: ${{ steps.build.outputs.deb_package_path != '' && github.event_name == 'workflow_dispatch' && inputs.create_release }}
         with:
           name: ${{ steps.build.outputs.deb_package_name }}
           path: ${{ steps.build.outputs.deb_package_path }}
       - name: Upload archive package artifacts
         uses: actions/upload-artifact@v4
-        if: ${{ steps.archive.outputs.archive_package_path != '' }}
+        if: ${{ steps.archive.outputs.archive_package_path != '' && github.event_name == 'workflow_dispatch' && inputs.create_release }}
         with:
           name: ${{ steps.archive.outputs.archive_package_name }}
           path: ${{ steps.archive.outputs.archive_package_path }}
+  build_extras:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, ubicloud-standard-4-arm]
+        postgres: [11, 12, 13, 14, 15, 16]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - name: Build
+        id: build
+        run: sudo su -c "PG_VERSION=$PG_VERSION SETUP_ENV=1 USE_SOURCE=1 SETUP_POSTGRES=1 PACKAGE_EXTENSION=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh"
+        env:
+          PG_VERSION: ${{ matrix.postgres }}
+      - name: Build Lantern CLI
+        id: build_cli
+        run: sudo su -c "PACKAGE_CLI=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/build-extras.sh"
+        if: ${{ matrix.postgres == 15 }} # run only once
+      - name: Upload archive package artifacts
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build.outputs.archive_package_name }}
+          path: ${{ steps.build.outputs.archive_package_path }}
+      - name: Upload Lantern CLI artifacts
+        if: ${{ matrix.postgres == 15 && github.event_name == 'workflow_dispatch' && inputs.create_release }} # run only once
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build_cli.outputs.cli_package_name }}
+          path: ${{ steps.build_cli.outputs.cli_package_path }}
   package:
     runs-on: ubuntu-22.04
-    needs: [build]
+    needs: [build_lantern, build_extras]
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: lantern-extras-*.tar
+          merge-multiple: true
+          path: /tmp/lantern-extras-package
+      - uses: geekyeggo/delete-artifact@v4
+        with:
+          name: lantern-extras*.tar
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: lantern-cli-*
+          merge-multiple: true
+          path: /tmp/lantern-cli-package
       - uses: actions/download-artifact@v4
         with:
           pattern: lantern-*.tar
@@ -73,24 +119,22 @@ jobs:
           name: ${{ steps.package.outputs.package_name }}
           path: ${{ steps.package.outputs.package_path }}
       - name: Create source code archive with submodules
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
         run: |
           find ./ -name '.git*' -exec rm -r {} \; || true
           tar -czf /tmp/lantern-v${{ steps.package.outputs.package_version }}-source.tar.gz .
       - name: Create GitHub release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         id: create_release
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
         with:
           name: Lantern v${{ steps.package.outputs.package_version }}
           tag_name: v${{ steps.package.outputs.package_version }}
           files: |
             ${{ steps.package.outputs.package_path }}
             /tmp/lantern-v${{ steps.package.outputs.package_version }}-source.tar.gz
+            /tmp/lantern-cli-package/*
           generate_release_notes: true
       - name: Homebrew release
         uses: var77/bump-homebrew-formula-action@main
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.create_release }}
         with:
           formula-name: lantern
           formula-path: Formula/lantern.rb

.github/workflows/publish-cli-docker.yaml

@@ -0,0 +1,170 @@
+name: publish-cli-docker
+on:
+  workflow_dispatch:
+    inputs:
+      LATEST:
+        type: boolean
+        description: "Publish as latest release"
+        required: false
+        default: false
+      VERSION:
+        type: string
+        description: "CLI version"
+        required: true
+        default: "0.3.27"
+      IMAGE_NAME:
+        type: string
+        description: "Container image name to tag"
+        required: true
+        default: "lanterndata/lantern-cli"
+jobs:
+  build-x64:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        device: [cpu, gpu]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GCR Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev
+          username: _json_key_base64
+          password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        id: build_image_latest
+        with:
+          context: .
+          platforms: |
+            linux/amd64
+          file: docker/Dockerfile.cli${{ (matrix.device == 'gpu' && '.cuda' || '') }}
+          push: true
+          # We will only correctly tag GPU image as we have it only for x86_64 platform
+          # CPU image will be tagged with github sha, to be merged later with arm64 image and pushed with correct tags
+          tags: |
+            ${{ (matrix.device == 'cpu' && format('{0}:{1}-cpu-amd64', inputs.IMAGE_NAME, github.sha) || format('{0}:gpu', inputs.IMAGE_NAME)) }}
+            ${{ (matrix.device == 'gpu' && format('{0}:{1}-gpu', inputs.IMAGE_NAME, inputs.VERSION)) || '' }}
+            ${{ (matrix.device == 'gpu' && (inputs.LATEST == true || inputs.LATEST == 'true') && format('{0}:latest-gpu', inputs.IMAGE_NAME)) || '' }}
+            ${{ (matrix.device == 'gpu' && (inputs.LATEST == true || inputs.LATEST == 'true') && format('{0}-docker.pkg.dev/{1}/{2}:latest-gpu', secrets.GCP_REGION, secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME)) || '' }}
+            ${{ (matrix.device == 'gpu' && format('{0}-docker.pkg.dev/{1}/{2}:{3}-gpu',secrets.GCP_REGION,secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME, inputs.VERSION)) || '' }}
+  build-x64-optimized:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [skylake, cascadelake, icelake-server]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GCR Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev
+          username: _json_key_base64
+          password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        id: build_optimized_image
+        with:
+          context: .
+          platforms: |
+            linux/amd64
+          build-args: |
+            CXXFLAGS=${{ format('-march={0} -DUSEARCH_USE_SIMSIMD=1 -DUSEARCH_USE_FP16LIB=0', matrix.target) }}
+          file: docker/Dockerfile.cli
+          push: true
+          tags: |
+            ${{ format('{0}:{1}-cpu-{2}', inputs.IMAGE_NAME, inputs.VERSION, matrix.target) }}
+            ${{ format('{0}-docker.pkg.dev/{1}/{2}:{3}-cpu-{4}',secrets.GCP_REGION,secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME, inputs.VERSION, matrix.target) }}
+  build-arm64:
+    runs-on: ubicloud-standard-4-arm
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push without latest tags
+        uses: docker/build-push-action@v5
+        id: build_image
+        with:
+          context: .
+          platforms: |
+            linux/arm64
+          file: docker/Dockerfile.cli
+          push: true
+          tags: |
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+  push:
+    runs-on: ubuntu-22.04
+    needs: [build-x64, build-arm64]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GCR Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev
+          username: _json_key_base64
+          password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }}
+      - name: Create latest tags and push
+        if: ${{ (inputs.LATEST == true || inputs.LATEST == 'true') }}
+        run: |
+          # :latest tag
+          docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:latest \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+          # :latest-cpu
+          docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:latest-cpu \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+          # :latest gcr
+          docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:latest \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+          # :latest-cpu gcr
+          docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:latest-cpu \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+      - name: Create and versioned tags and push
+        run: |
+          # :version-cpu tag
+          docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:${{ inputs.VERSION }}-cpu \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64
+          # :version-cpu gcr
+          docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:${{ inputs.VERSION }}-cpu \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \
+            ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64

.github/workflows/sanitizer-build-and-test.yaml

@@ -33,16 +33,16 @@ env:
   CXXFLAGS: "-g -fsanitize=address -fno-omit-frame-pointer -Og -fno-inline-functions"
   LDFLAGS: "-fsanitize=address"
 
-  ASAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/sanitizers/suppressions/suppr_asan.txt
+  ASAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_asan.txt
     detect_odr_violation=0 log_path=${{ github.workspace }}/sanitizer/
     log_exe_name=true print_suppressions=false exitcode=27
     detect_leaks=0 abort_on_error=1
 
-  LSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/sanitizers/suppressions/suppr_leak.txt
+  LSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_leak.txt
     print_suppressions=0 log_path=${{ github.workspace }}/sanitizer/
     log_exe_name=true print_suppressions=false exitcode=27
 
-  UBSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/sanitizers/suppressions/suppr_ub.txt
+  UBSAN_OPTIONS: suppressions=${{ github.workspace }}/lantern_hnsw/scripts/sanitizers/suppressions/suppr_ub.txt
     print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer/
     log_exe_name=true print_suppressions=false exitcode=27
 
@@ -127,7 +127,7 @@ jobs:
         tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1
         # Add instrumentation to the Postgres memory contexts. For more details, see
         # https://www.postgresql.org/message-id/CAM-w4HNH7%2BU9jZevpVK7Wr49tkfpWSR6wav0RLYrq0HWuP5cxw%40mail.gmail.com
-        patch -F5 -p1 -d ~/$PG_SRC_DIR < scripts/sanitizers/postgres-asan-instrumentation.patch
+        patch -F5 -p1 -d ~/$PG_SRC_DIR < lantern_hnsw/scripts/sanitizers/postgres-asan-instrumentation.patch
         cd ~/$PG_SRC_DIR
         ./configure --prefix=$PG_INSTALL_DIR --enable-debug --enable-cassert \
           --with-openssl --without-readline --without-zlib --without-libxml
@@ -160,8 +160,8 @@ jobs:
 
     - name: Build Lantern
       run: |
-        mkdir lantern_build
-        cd lantern_build
+        mkdir lantern_hnsw/lantern_build
+        cd lantern_hnsw/lantern_build
         CXXFLAG="" cmake ..
         make install
 
@@ -177,7 +177,7 @@ jobs:
 
     - name: make test
       run: |
-        cd lantern_build
+        cd lantern_hnsw/lantern_build
         make test
 
     - name: Show regression diffs
@@ -229,7 +229,7 @@ jobs:
           info locals
           bt full
         " 2>&1 | tee stacktrace.log
-        ./scripts/sanitizers/bundle_coredump.sh
+        ./lantern_hnsw/scripts/sanitizers/bundle_coredump.sh
         grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||:
 
     - name: Coredumps