From f1ba7323cbd7b1241626ae131865fe7346626c36 Mon Sep 17 00:00:00 2001 From: Danny Browning Date: Tue, 27 Feb 2024 14:18:10 -0700 Subject: [PATCH] fix: add certs to container and check for connections in build (#23) * fix: add certs to container and check for connections in build * fix: github and docker just don't play nice --- .github/workflows/build-and-test.yml | 17 ++++++++++- .github/workflows/publish.yml | 1 + Cargo.lock | 4 +-- Cargo.toml | 2 ++ Dockerfile | 2 ++ Makefile | 16 +++++++--- checkpointer/Cargo.toml | 3 +- checkpointer/src/batcher.rs | 2 +- checkpointer/src/errors.rs | 2 ++ checkpointer/src/main.rs | 44 ++++++++++++++++++++++++---- checkpointer/src/persistence.rs | 2 ++ ci-scripts/image.sh | 11 +++++++ ci-scripts/publish.sh | 4 +-- ci-scripts/setup_test_env.sh | 7 ++++- it/docker-compose.yml | 3 +- tester/Cargo.toml | 3 +- 16 files changed, 102 insertions(+), 21 deletions(-) create mode 100755 ci-scripts/image.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5c2ac97..9f26d1d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -46,7 +46,7 @@ jobs: - uses: mozilla-actions/sccache-action@v0.0.3 - name: Run tests run: | - echo "Skipping tests" + echo "Skipping tests since github won't let us use tmp directories" env: DID_DOCUMENT: ${{ vars.DID_DOCUMENT }} DID_PRIVATE_KEY: ${{ secrets.DID_PRIVATE_KEY }} @@ -67,3 +67,18 @@ jobs: - uses: mozilla-actions/sccache-action@v0.0.3 - name: Build run: make build + build-docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build Image + run: | + make build-docker + env: + DID_DOCUMENT: ${{ vars.DID_DOCUMENT }} + DID_PRIVATE_KEY: ${{ secrets.DID_PRIVATE_KEY }} + RUST_LOG: info + CERAMIC_URL: ${{ vars.CERAMIC_URL }} + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d0991f7..8f77312 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -30,6 +30,7 @@ jobs: password: ${{ env.AWS_SECRET_ACCESS_KEY }} env: AWS_REGION: us-east-1 + CERAMIC_URL: ${{ vars.CERAMIC_URL }} - name: Build and Publish id: build diff --git a/Cargo.lock b/Cargo.lock index a5cc7ae..7cc0c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1060,7 +1060,8 @@ dependencies = [ "calculator", "ceramic-http-client", "chrono", - "curve25519-dalek", + "clap", + "env_logger", "futures-util", "models", "reqwest", @@ -6551,7 +6552,6 @@ dependencies = [ "ceramic-http-client", "chrono", "clap", - "curve25519-dalek", "models", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 36fce80..630c6dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ members = [ anyhow = "1.0.80" async-trait = "0.1.77" base64 = "0.21.7" + clap = { version = "=4.4.18", default-features = false, features = ["derive", "std"] } + curve25519-dalek = "=4.1.1" hmac = "0.12.1" jwt = "0.16.0" log = "0.4.20" diff --git a/Dockerfile b/Dockerfile index f2c2397..543fd48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,8 @@ RUN --mount=type=cache,target=/home/builder/.cargo \ FROM --platform=linux/amd64 debian:bookworm-slim +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + COPY --from=builder /home/builder/checkpointer/bin/* /usr/bin # Adding this step after copying the ceramic-one binary so that we always take the newest libs from the builder if the diff --git a/Makefile b/Makefile index b09a1f0..e212d69 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,9 @@ DEPLOY_TAG ?= latest # Whether or not this is a manual deployment MANUAL_DEPLOY ?= false -DATABASE_URL ?= sqlite://checkpointer.db +RUST_LOG ?= info + +.EXPORT_ALL_VARIABLES: .PHONY: all all: build check-fmt check-clippy test @@ -57,9 +59,9 @@ test: # Setup scaffolding ./ci-scripts/setup_test_env.sh # Test with default features - DATABASE_URL=$(DATABASE_URL) $(CARGO) test -p checkpointer --locked --release + RUST_LOG=$(RUST_LOG) $(CARGO) test -p checkpointer --locked --release # Test with all features - DATABASE_URL=$(DATABASE_URL) $(CARGO) test -p checkpointer --locked --release --all-features + RUST_LOG=$(RUST_LOG) $(CARGO) test -p checkpointer --locked --release --all-features ./ci-scripts/teardown_test_env.sh .PHONY: test-event-joiner @@ -87,8 +89,14 @@ check-clippy: run: RUST_LOG=WARN,checkpointer=DEBUG $(CARGO) run --all-features --locked --release --bin checkpointer +.PHONY: build-docker +build-docker: + ./ci-scripts/setup_test_env.sh + ./ci-scripts/image.sh + ./ci-scripts/teardown_test_env.sh + .PHONY: publish-docker -publish-docker: +publish-docker: build-docker ./ci-scripts/publish.sh .PHONY: schedule-ecs-deployment diff --git a/checkpointer/Cargo.toml b/checkpointer/Cargo.toml index 496ee36..ca6175c 100644 --- a/checkpointer/Cargo.toml +++ b/checkpointer/Cargo.toml @@ -13,6 +13,7 @@ async-trait = "0.1.77" calculator = { path = "../calculator" } ceramic-http-client = { workspace = true, features = ["remote"] } chrono.workspace = true +clap.workspace = true futures-util = "0.3.30" models = { path = "../models" } reqwest = "0.11.23" @@ -29,6 +30,6 @@ url.workspace = true util = { path = "../util" } [dev-dependencies] -curve25519-dalek = "=4.1.1" +env_logger = "0.11.2" tmpdir = "1.0.0" wiremock = "0.6.0-rc.2" diff --git a/checkpointer/src/batcher.rs b/checkpointer/src/batcher.rs index 12728da..b4e2724 100644 --- a/checkpointer/src/batcher.rs +++ b/checkpointer/src/batcher.rs @@ -290,7 +290,7 @@ mod tests { #[tokio::test] async fn should_receive_create_and_update_events() { - let _guard = util::init_tracing(); + let _ = env_logger::try_init(); let ceramic_url = ceramic_url(); let db = Arc::new(InMemoryPersistence::new()); diff --git a/checkpointer/src/errors.rs b/checkpointer/src/errors.rs index 369de05..b7fad84 100644 --- a/checkpointer/src/errors.rs +++ b/checkpointer/src/errors.rs @@ -22,6 +22,8 @@ pub enum Error { #[error("{0}")] Ceramic(#[from] anyhow::Error), #[error("{0}")] + Reqwest(#[from] reqwest::Error), + #[error("{0}")] Custom(String), } diff --git a/checkpointer/src/main.rs b/checkpointer/src/main.rs index 34d50b9..27d103c 100644 --- a/checkpointer/src/main.rs +++ b/checkpointer/src/main.rs @@ -16,10 +16,25 @@ mod persistence; use crate::persistence::SqlitePersistence; use batcher::{BatchCreationParameters, Batcher}; use calculator::CalculatorParameters; +use clap::{Parser, Subcommand}; use errors::Error; use std::sync::atomic::AtomicBool; use std::sync::Arc; +#[derive(Parser)] +#[command(name = "CeramicCheckpointer")] +#[command(version = "1.0")] +#[command(about = "Provides batching and checkpointing for ceramic sse feeds", long_about = None)] +struct Cli { + #[clap(subcommand)] + subcmd: Option, +} + +#[derive(Subcommand)] +enum SubCmd { + SshCheck, +} + fn trace_error(res: ServiceResponse) -> Result> { if let Some(ref e) = res.response().error() { tracing::error!("{:?}", e); @@ -106,13 +121,32 @@ pub struct Config { #[actix_web::main] async fn main() -> Result<(), Error> { let _guard = util::init_tracing(); + let cmd = Cli::parse(); - let config = Config { - batcher: Batcher::new(Arc::new(SqlitePersistence::new().await?))?, - calculator_params: CalculatorParameters::new().await?, - calculate_active: Arc::new(AtomicBool::new(false)), - }; + let calculator_params = CalculatorParameters::new().await?; + match cmd.subcmd { + Some(SubCmd::SshCheck) => { + let url = calculator_params + .ceramic_url + .join("/api/v0/node/healthcheck")?; + if !reqwest::get(url).await?.status().is_success() { + return Err(Error::custom("Failed to connect to ceramic")); + } + } + None => { + let config = Config { + batcher: Batcher::new(Arc::new(SqlitePersistence::new().await?))?, + calculator_params, + calculate_active: Arc::new(AtomicBool::new(false)), + }; + start_server(config).await?; + } + } + + Ok(()) +} +async fn start_server(config: Config) -> Result<(), Error> { HttpServer::new(move || { let svc = web::scope("/api/v1") .service(create_batcher) diff --git a/checkpointer/src/persistence.rs b/checkpointer/src/persistence.rs index 9de1736..f6d1c86 100644 --- a/checkpointer/src/persistence.rs +++ b/checkpointer/src/persistence.rs @@ -108,6 +108,8 @@ pub mod tests { #[tokio::test] async fn can_add_and_retrieve_events() { + let _ = env_logger::try_init(); + let pool = setup().await; let client_id = "test_client"; let event = Event { diff --git a/ci-scripts/image.sh b/ci-scripts/image.sh new file mode 100755 index 0000000..bb843e9 --- /dev/null +++ b/ci-scripts/image.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if [ -z "$DID_DOCUMENT" ]; then + echo "No DID_DOCUMENT specified, cannot verify image" + exit 1 +fi + +# Build a docker image running checkpointer +docker buildx build --load -t 3box/checkpointer . +#docker run -e DID_DOCUMENT=$DID_DOCUMENT -e DID_PRIVATE_KEY=$DID_PRIVATE_KEY -e CERAMIC_URL=$CERAMIC_URL --rm 3box/checkpointer ssh-check +docker run 3box/checkpointer --version \ No newline at end of file diff --git a/ci-scripts/publish.sh b/ci-scripts/publish.sh index b5810c8..e114ad6 100755 --- a/ci-scripts/publish.sh +++ b/ci-scripts/publish.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Build and publish a docker image run running checkpointer +# Publish a docker image running checkpointer # # DOCKER_PASSWORD must be set # Use: @@ -9,8 +9,6 @@ # echo "${DOCKER_PASSWORD}" | docker login --username AWS --password-stdin public.ecr.aws/r5b3e0r5 # # to login to docker. That password will be valid for 12h. - -docker buildx build --load -t 3box/checkpointer . docker tag 3box/checkpointer:latest public.ecr.aws/r5b3e0r5/3box/checkpointer:latest if [[ -n "$SHA" ]]; then diff --git a/ci-scripts/setup_test_env.sh b/ci-scripts/setup_test_env.sh index 9af9ef9..4656f76 100755 --- a/ci-scripts/setup_test_env.sh +++ b/ci-scripts/setup_test_env.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash docker compose -f it/docker-compose.yml up -d ceramic +if [ -z "$DID_DOCUMENT" ]; then + echo "No DID_DOCUMENT specified, cannot setup environment" + exit 1 +fi + echo "Starting ceramic" while [ $(curl -s -o /dev/null -I -w "%{http_code}" "http://localhost:7007/api/v0/node/healthcheck") -ne "200" ]; do echo "Ceramic is not yet ready, waiting and trying again" sleep 1 done -if [ -z "$IT_TEST_CHECKPOINTER" ]; then +if [ -n "${IT_TEST_CHECKPOINTER}" ]; then echo "Starting Checkpointer" mkdir it/sqlite docker compose -f it/docker-compose.yml up -d checkpointer diff --git a/it/docker-compose.yml b/it/docker-compose.yml index 6a4b249..bd17b9f 100644 --- a/it/docker-compose.yml +++ b/it/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: ceramic: - image: ceramicnetwork/js-ceramic:dev + image: ceramicnetwork/js-ceramic:5.2.0 volumes: - ./data:/root/.ceramic ports: @@ -15,6 +15,7 @@ services: - RUST_LOG=info - RUST_BACKTRACE=1 - CERAMIC_URL=http://ceramic-service:7007 + - DID_DOCUMENT=${DID_DOCUMENT} - DID_PRIVATE_KEY=${DID_PRIVATE_KEY} - DATABASE_URL=${DATABASE_URL} depends_on: diff --git a/tester/Cargo.toml b/tester/Cargo.toml index 1287e40..0236bb1 100644 --- a/tester/Cargo.toml +++ b/tester/Cargo.toml @@ -9,8 +9,7 @@ edition = "2021" anyhow = "1.0.79" ceramic-http-client = { workspace = true, features = ["remote"] } chrono.workspace = true -clap = { version = "=4.4.18", default-features = false, features = ["derive", "std"] } -curve25519-dalek = "=4.1.1" +clap.workspace = true models = { path = "../models" } serde.workspace = true serde_json.workspace = true