Tests

Extract Python components into their own packages #367

Workflow file for this run

	name: Tests

	on:
	push:
	branches:
	- 'main'
	pull_request:
	types: [opened, synchronize, reopened, labeled, unlabeled]
	branches:
	- 'main'
	paths:
	- 'server/**'
	- 'executor/**'
	- 'function_executor/**'
	- 'indexify/**'
	- 'python_utils/**'
	- 'python-sdk/**'
	- '.github/workflows/tests.yaml'

	env:
	CARGO_TERM_COLOR: always
	INDEXIFY_URL: http://localhost:8900

	jobs:
	lint_server:
	name: Lint Indexify Server
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Lint indexify-server
	run: \|
	cd server
	make check

	build_server:
	name: Build Indexify Server
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Copy rust-toolchain
	run: cp server/rust-toolchain.toml .
	- name: Setup Rust
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache-directories: \|
	server/target
	- name: Setup Rust
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache-directories: \|
	server/target

	- name: Build indexify-server
	run: \|
	cd server
	cargo build
	- name: Lint indexify-server
	run: \|
	cd server
	make check
	- name: Test indexify-server
	run: \|
	cd server
	cargo test --workspace -- --test-threads 1
	- name: Upload indexify-server
	uses: actions/upload-artifact@v4
	with:
	name: indexify-server
	path: server/target/debug/indexify-server

	lint_python_packages:
	name: Lint Python packages
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Install poetry
	run: pipx install poetry
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: 3.11
	cache: 'poetry'
	- name: Build executor
	run: cd executor && make build
	- name: Lint executor
	run: cd executor && make check
	- name: Build function_executor
	run: cd function_executor && make build
	- name: Lint function_executor
	run: cd function_executor && make check
	- name: Build indexify
	run: cd indexify && make build
	- name: Lint indexify
	run: cd indexify && make check
	- name: Build python_utils
	run: cd python_utils && make build
	- name: Lint python_utils
	run: cd python_utils && make check
	- name: Build python-sdk
	run: cd python-sdk && make build
	- name: Lint python-sdk
	run: cd python-sdk && make check

	acceptance_tests:
	name: Run Acceptance Tests
	# needs: [build_server, lint_python_packages]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	# - name: Download indexify-server
	# uses: actions/download-artifact@v4
	# with:
	# name: indexify-server
	- name: Install poetry
	run: pipx install poetry
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: 3.11
	cache: 'poetry'
	- name: Build indexify
	run: \|
	cd indexify
	make build
	- name: Lint indexify
	run: \|
	cd indexify
	make check
	# - name: Start Background Indexify Server
	# uses: JarvusInnovations/background-action@v1
	# with:
	# run: \|
	# chmod u+x ./indexify-server
	# RUST_LOG=debug ./indexify-server &
	# echo $! > /tmp/indexify-server.pid &

	# wait-on: \|
	# tcp:localhost:8900

	# tail: true
	# wait-for: 30s
	# log-output: true
	# # always logging the output to debug test failures.
	# log-output-if: true
	- name: Start Indexify Executor
	run: \|
	ls
	cd indexify
	`poetry env list --full-path \| awk '{ print $1 }'`/bin/indexify-cli
	strace -f poetry run indexify-cli
	poetry run indexify-cli --help
	poetry run indexify-cli executor
	# ls
	# pwd
	# poetry debug info
	# make check
	# poetry run python -c "print('python version:', __import__('sys').version)"
	# poetry run python -c "import python_utils"
	# echo "python_utils imported"
	# poetry run python -c "import function_executor"
	# echo "function_executor imported"
	# poetry run python -c "import executor"
	# echo "executor imported"
	# poetry run python -c "import indexify"
	# echo "indexify imported"
	# poetry run python -c "import cli"
	# echo "cli imported"
	# ls -l `poetry env list --full-path \| awk '{ print $1 }'`/bin/indexify-cli
	# file `poetry env list --full-path \| awk '{ print $1 }'`/bin/indexify-cli
	# cat `poetry env list --full-path \| awk '{ print $1 }'`/bin/indexify-cli
	# ls -l `poetry env list --full-path \| awk '{ print $1 }'`/bin/
	# ls -l `poetry env list --full-path \| awk '{ print $1 }'`/lib/python3.11/site-packages/
	- name: Start Background Indexify Executor
	uses: JarvusInnovations/background-action@v1
	with:
	run: \|
	cd indexify
	poetry run indexify-cli executor &
	echo $! > /tmp/indexify-executor.pid &

	wait-on: \|
	tcp:localhost:8900

	tail: true
	wait-for: 10s
	log-output: true
	# always logging the output to debug test failures.
	log-output-if: true
	- name: Wait for readiness
	run: \|
	serverReady=false
	counter=0
	while [ "$serverReady" != true ]; do
	output=$(curl --silent --fail http://localhost:8900/internal/executors \| jq '. \| length' 2>/dev/null)
	if [[ $? -eq 0 && "$output" -ge 1 ]]; then
	echo "Server ready with executors."
	serverReady=true
	else
	echo 'Waiting for executors to join server...'
	counter=$((counter+1))
	if [ $counter -gt 6 ]; then
	echo "Timeout waiting for executors to join server."
	exit 1
	fi
	sleep 5
	fi
	done
	- name: Run executor tests
	run: cd executor && make test
	- name: Run function_executor tests
	run: cd function_executor && make test
	- name: Run indexify tests
	run: cd indexify && make test
	- name: Run python_utils tests
	run: cd python_utils && make test
	- name: Run python-sdk tests
	run: cd python-sdk && make test
	- name: Terminate processes
	# We want to test clean termination of processes.
	run: \|
	pkill -SIGTERM -F /tmp/indexify-server.pid
	pkill -SIGTERM -F /tmp/indexify-executor.pid
	- name: Wait for processes termination
	run: \|
	pid_files="/tmp/indexify-server.pid /tmp/indexify-executor.pid"
	for pid_file in $pid_files; do
	while pkill -0 -F "$pid_file" 2>/dev/null; do
	echo "waiting for process $pid_file to exit..."
	sleep 1
	done
	done
	- name: Validate that all processes terminated
	run: \|
	if pgrep -f indexify; then
	echo "Error: Some Indexify processes are still running."
	exit 1
	fi

	last_release_acceptance_tests:
	name: 'Last Release Acceptance Tests (trigger with label: ci_compat_test)'
	if: contains(github.event.pull_request.labels.*.name, 'ci_compat_test')
	needs: [build_server, lint_python_packages]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Checkout latest
	run: \|
	git fetch --tags
	latestTag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
	git checkout $latestTag
	- name: Download indexify-server
	uses: actions/download-artifact@v4
	with:
	name: indexify-server
	- name: Start Background Indexify Server
	uses: JarvusInnovations/background-action@v1
	with:
	run: \|
	chmod u+x ./indexify-server
	RUST_LOG=debug ./indexify-server &
	echo $! > /tmp/indexify-server.pid &

	wait-on: \|
	tcp:localhost:8900

	tail: true
	wait-for: 30s
	log-output: true
	# always logging the output to debug test failures.
	log-output-if: true
	- name: Install poetry
	run: pipx install poetry
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: 3.11
	cache: 'poetry'
	- name: Build indexify
	run: cd indexify && make build
	- name: Start Background Indexify Executor
	uses: JarvusInnovations/background-action@v1
	with:
	run: \|
	cd indexify
	poetry run indexify-cli executor &
	echo $! > /tmp/indexify-executor.pid &

	wait-on: \|
	tcp:localhost:8900

	tail: true
	wait-for: 10s
	log-output: true
	# always logging the output to debug test failures.
	log-output-if: true
	- name: Wait for readiness
	run: \|
	serverReady=false
	counter=0
	while [ "$serverReady" != true ]; do
	output=$(curl --silent --fail http://localhost:8900/internal/executors \| jq '. \| length' 2>/dev/null)
	if [[ $? -eq 0 && "$output" -ge 1 ]]; then
	echo "Server ready with executors."
	serverReady=true
	else
	echo 'Waiting for executors to join server...'
	counter=$((counter+1))
	if [ $counter -gt 6 ]; then
	echo "Timeout waiting for executors to join server."
	exit 1
	fi
	sleep 5
	fi
	done
	- name: Run executor tests
	run: cd executor && make test
	- name: Run function_executor tests
	run: cd function_executor && make test
	- name: Run indexify tests
	run: cd indexify && make test
	- name: Run python_utils tests
	run: cd python_utils && make test
	- name: Run python-sdk tests
	run: cd python-sdk && make test
	- name: Terminate processes
	# We want to test clean termination of processes.
	run: \|
	pkill -SIGTERM -F /tmp/indexify-server.pid
	pkill -SIGTERM -F /tmp/indexify-executor.pid
	- name: Wait for processes termination
	run: \|
	pid_files="/tmp/indexify-server.pid /tmp/indexify-executor.pid"
	for pid_file in $pid_files; do
	while pkill -0 -F "$pid_file" 2>/dev/null; do
	echo "waiting for process $pid_file to exit..."
	sleep 1
	done
	done
	- name: Validate that all processes terminated
	run: \|
	if pgrep -f indexify; then
	echo "Error: Some Indexify processes are still running."
	exit 1
	fi

	build_release_packages:
	name: Test Building Release Packages
	needs: [build_server]
	uses: ./.github/workflows/wf_build_indexify_server_release_packages.yaml

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Extract Python components into their own packages #367

Workflow file

Extract Python components into their own packages #367

Jobs

Run details

Workflow file for this run