-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #339 from lanterndata/varik/merge-lantern-extras
Move lantern_extras to lantern repo
- Loading branch information
Showing
298 changed files
with
17,958 additions
and
250 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| [target.aarch64-apple-darwin] | ||
| rustflags = ["-C", "link-args=-Wl,-rpath,/usr/local/lib/onnxruntime/lib"] | ||
| [target.x86_64-apple-darwin] | ||
| rustflags = ["-C", "link-args=-Wl,-rpath,/usr/local/lib/onnxruntime/lib"] | ||
| [target.'cfg(target_os="macos")'] | ||
| # Postgres symbols won't be available until runtime | ||
| rustflags = ["-Clink-arg=-Wl,-undefined,dynamic_lookup"] | ||
| [env] | ||
| ORT_STRATEGY = { value = "system" } | ||
| ORT_LIB_LOCATION = { value = "/usr/local/lib/onnxruntime/lib" , relative = false } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| .git | ||
| .cargo | ||
| .github | ||
| target | ||
| .DS_Store | ||
| lantern_hnsw/build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| name: publish-cli-docker | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| LATEST: | ||
| type: boolean | ||
| description: "Publish as latest release" | ||
| required: false | ||
| default: false | ||
| VERSION: | ||
| type: string | ||
| description: "CLI version" | ||
| required: true | ||
| default: "0.3.27" | ||
| IMAGE_NAME: | ||
| type: string | ||
| description: "Container image name to tag" | ||
| required: true | ||
| default: "lanterndata/lantern-cli" | ||
| jobs: | ||
| build-x64: | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| device: [cpu, gpu] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to DockerHub | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| - name: Login to GCR Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev | ||
| username: _json_key_base64 | ||
| password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }} | ||
| - name: Build and push | ||
| uses: docker/build-push-action@v5 | ||
| id: build_image_latest | ||
| with: | ||
| context: . | ||
| platforms: | | ||
| linux/amd64 | ||
| file: docker/Dockerfile.cli${{ (matrix.device == 'gpu' && '.cuda' || '') }} | ||
| push: true | ||
| # We will only correctly tag GPU image as we have it only for x86_64 platform | ||
| # CPU image will be tagged with github sha, to be merged later with arm64 image and pushed with correct tags | ||
| tags: | | ||
| ${{ (matrix.device == 'cpu' && format('{0}:{1}-cpu-amd64', inputs.IMAGE_NAME, github.sha) || format('{0}:gpu', inputs.IMAGE_NAME)) }} | ||
| ${{ (matrix.device == 'gpu' && format('{0}:{1}-gpu', inputs.IMAGE_NAME, inputs.VERSION)) || '' }} | ||
| ${{ (matrix.device == 'gpu' && (inputs.LATEST == true || inputs.LATEST == 'true') && format('{0}:latest-gpu', inputs.IMAGE_NAME)) || '' }} | ||
| ${{ (matrix.device == 'gpu' && (inputs.LATEST == true || inputs.LATEST == 'true') && format('{0}-docker.pkg.dev/{1}/{2}:latest-gpu', secrets.GCP_REGION, secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME)) || '' }} | ||
| ${{ (matrix.device == 'gpu' && format('{0}-docker.pkg.dev/{1}/{2}:{3}-gpu',secrets.GCP_REGION,secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME, inputs.VERSION)) || '' }} | ||
| build-x64-optimized: | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| target: [skylake, cascadelake, icelake-server] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to DockerHub | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| - name: Login to GCR Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev | ||
| username: _json_key_base64 | ||
| password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }} | ||
| - name: Build and push | ||
| uses: docker/build-push-action@v5 | ||
| id: build_optimized_image | ||
| with: | ||
| context: . | ||
| platforms: | | ||
| linux/amd64 | ||
| build-args: | | ||
| CXXFLAGS=${{ format('-march={0} -DUSEARCH_USE_SIMSIMD=1 -DUSEARCH_USE_FP16LIB=0', matrix.target) }} | ||
| file: docker/Dockerfile.cli | ||
| push: true | ||
| tags: | | ||
| ${{ format('{0}:{1}-cpu-{2}', inputs.IMAGE_NAME, inputs.VERSION, matrix.target) }} | ||
| ${{ format('{0}-docker.pkg.dev/{1}/{2}:{3}-cpu-{4}',secrets.GCP_REGION,secrets.GCP_PROJECT_ID, inputs.IMAGE_NAME, inputs.VERSION, matrix.target) }} | ||
| build-arm64: | ||
| runs-on: ubicloud-standard-4-arm | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to DockerHub | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| - name: Build and push without latest tags | ||
| uses: docker/build-push-action@v5 | ||
| id: build_image | ||
| with: | ||
| context: . | ||
| platforms: | | ||
| linux/arm64 | ||
| file: docker/Dockerfile.cli | ||
| push: true | ||
| tags: | | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| push: | ||
| runs-on: ubuntu-22.04 | ||
| needs: [build-x64, build-arm64] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to DockerHub | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
| password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
| - name: Login to GCR Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ${{ secrets.GCP_REGION }}-docker.pkg.dev | ||
| username: _json_key_base64 | ||
| password: ${{ secrets.GCP_CREDENTIALS_JSON_B64 }} | ||
| - name: Create latest tags and push | ||
| if: ${{ (inputs.LATEST == true || inputs.LATEST == 'true') }} | ||
| run: | | ||
| # :latest tag | ||
| docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:latest \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| # :latest-cpu | ||
| docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:latest-cpu \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| # :latest gcr | ||
| docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:latest \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| # :latest-cpu gcr | ||
| docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:latest-cpu \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| - name: Create and versioned tags and push | ||
| run: | | ||
| # :version-cpu tag | ||
| docker buildx imagetools create -t ${{ inputs.IMAGE_NAME }}:${{ inputs.VERSION }}-cpu \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 | ||
| # :version-cpu gcr | ||
| docker buildx imagetools create -t ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ inputs.IMAGE_NAME }}:${{ inputs.VERSION }}-cpu \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-amd64 \ | ||
| ${{ inputs.IMAGE_NAME }}:${{ github.sha }}-cpu-arm64 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.