diff --git a/Dockerfile b/Dockerfile index 2ecace33..bc626e68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,4 @@ COPY ./backend . # Copy the frontend build COPY --from=builder /frontend/dist ./ui -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --start-interval=1s --retries=3 CMD [ "curl", "-f", "http://localhost:8000/health" ] - ENTRYPOINT [ "uvicorn", "app.server:app", "--host", "0.0.0.0" ] diff --git a/README.md b/README.md index 240ad8d7..d731d482 100644 --- a/README.md +++ b/README.md @@ -174,40 +174,7 @@ Navigate to [http://localhost:5173/](http://localhost:5173/) and enjoy! ## Migrating data from Redis to Postgres -OpenGPTs previously used Redis for data persistence, but has since switched to Postgres. If you have data in Redis that you would like to migrate to Postgres, you can use the following steps: - -### With Docker - -Add Postgres' environment variables to the `.env` file (by following `.env.example`). Then, start the services with `docker compose` and run the following command to migrate data from Redis to Postgres: - -```shell -docker exec -it opengpts-backend make redis_to_postgres -``` - -### Without Docker - -Make sure the following environment variables are set: - -```shell -export POSTGRES_HOST=... -export POSTGRES_PORT=... -export POSTGRES_DB=... -export POSTGRES_USER=... -export POSTGRES_PASSWORD=... -export REDIS_URL=... -``` - -Install [golang-migrate](https://github.com/golang-migrate/migrate) on your machine if you haven't already and run database schema migrations: - -```shell -make migrate -``` - -Finally, run the following command to migrate data from Redis to Postgres: - -```shell -make redis_to_postgres -``` +Refer to this [guide](tools/redis_to_postgres/README.md) for migrating data from Redis to Postgres. ## Features diff --git a/backend/Makefile b/backend/Makefile index cae39759..3e8dc5b2 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -19,9 +19,6 @@ start: migrate: migrate -database postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DB)?sslmode=disable -path ./migrations up -redis_to_postgres: - poetry run python redis_to_postgres.py - test: # We need to update handling of env variables for tests YDC_API_KEY=placeholder OPENAI_API_KEY=placeholder poetry run pytest $(TEST_FILE) diff --git a/backend/poetry.lock b/backend/poetry.lock index 58cb8e0e..9ff7cde6 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -2994,24 +2994,6 @@ files = [ [package.extras] full = ["numpy"] -[[package]] -name = "redis" -version = "5.0.1" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.7" -files = [ - {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"}, - {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} - -[package.extras] -hiredis = ["hiredis (>=1.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] - [[package]] name = "regex" version = "2023.10.3" @@ -4184,4 +4166,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "bc7c67284c2b95f2880a5bd9a1a6bff68ddb7484bfa04013874e9166899520e2" +content-hash = "6aba3d05838348cb038b98803ed0d66e70caaa08a56d5a16fdbea3c5c63ec073" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 189e799f..148ef410 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -43,7 +43,6 @@ pgvector = "^0.2.5" psycopg2-binary = "^2.9.9" asyncpg = "^0.29.0" langchain-core = "^0.1.33" -redis = "5.0.1" [tool.poetry.group.dev.dependencies] uvicorn = "^0.23.2" diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index cf419b07..e94ac4ac 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -1,13 +1,6 @@ version: "3" services: - redis: - container_name: opengpts-redis - image: redis/redis-stack-server:latest - ports: - - "6379:6379" - volumes: - - ./redis-volume:/data postgres: image: pgvector/pgvector:pg16 healthcheck: @@ -44,4 +37,3 @@ services: - .env environment: POSTGRES_HOST: "postgres" - REDIS_URL: "redis://opengpts-redis:6379" diff --git a/docker-compose.yml b/docker-compose.yml index c4000a98..6d59cb42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,6 @@ version: "3" services: - redis: - container_name: opengpts-redis - image: redis/redis-stack-server:latest - ports: - - "6379:6379" - volumes: - - ./redis-volume:/data postgres: image: pgvector/pgvector:pg16 healthcheck: @@ -47,7 +40,6 @@ services: - ./backend:/backend environment: POSTGRES_HOST: "postgres" - REDIS_URL: "redis://opengpts-redis:6379" command: - --reload frontend: diff --git a/tools/redis_to_postgres/Dockerfile b/tools/redis_to_postgres/Dockerfile new file mode 100644 index 00000000..7055882a --- /dev/null +++ b/tools/redis_to_postgres/Dockerfile @@ -0,0 +1,8 @@ +FROM langchain/open-gpts:latest + +RUN poetry add redis==5.0.1 + +COPY migrate_data.py . + +# Run database schema migrations and then migrate data +ENTRYPOINT sh -c "make migrate && python migrate_data.py" \ No newline at end of file diff --git a/tools/redis_to_postgres/README.md b/tools/redis_to_postgres/README.md new file mode 100644 index 00000000..869adecb --- /dev/null +++ b/tools/redis_to_postgres/README.md @@ -0,0 +1,9 @@ +OpenGPTs previously used Redis for data persistence, but has since switched to Postgres. If you have data in Redis that you would like to migrate to Postgres, follow the instructions below. + +Navigate to the `tools/redis_to_postgres` directory and ensure that the environment variables in the docker-compose file are set correctly for your Redis and Postgres instances. Then, run the following command to perform the migration: + +```shell +docker compose up --build --abort-on-container-exit +``` + +This will run database schema migrations for Postgres and then copy data from Redis to Postgres. Eventually all containers will be stopped. \ No newline at end of file diff --git a/tools/redis_to_postgres/docker-compose.yml b/tools/redis_to_postgres/docker-compose.yml new file mode 100644 index 00000000..d15cac22 --- /dev/null +++ b/tools/redis_to_postgres/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3" + +services: + redis: + container_name: opengpts-redis + image: redis/redis-stack-server:latest + ports: + - "6380:6379" + volumes: + - ./../../redis-volume:/data + data-migrator: + build: + context: . + depends_on: + - redis + network_mode: "host" + environment: + REDIS_URL: "redis://localhost:6380" + POSTGRES_HOST: "localhost" + POSTGRES_PORT: 5433 + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + OPENAI_API_KEY: ... diff --git a/backend/redis_to_postgres.py b/tools/redis_to_postgres/migrate_data.py similarity index 100% rename from backend/redis_to_postgres.py rename to tools/redis_to_postgres/migrate_data.py