Skip to content

Commit

Permalink
add debug logs to prin COPY time for embedding jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
var77 committed Nov 20, 2024
1 parent 4ab3297 commit 88ad71f
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 2 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/publish-cli-docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
type: string
description: "CLI version"
required: true
default: "0.4.0"
default: "0.5.1"
IMAGE_NAME:
type: string
description: "Container image name to tag"
Expand Down
2 changes: 1 addition & 1 deletion lantern_cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "lantern_cli"
version = "0.5.0"
version = "0.5.1"
edition = "2021"

[[bin]]
Expand Down
8 changes: 8 additions & 0 deletions lantern_cli/src/embeddings/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -512,13 +512,18 @@ async fn db_exporter_worker(
// if job is run in streaming mode
// it will write results to target table each 10 seconds (if collected rows are
// more than 50) or if collected row count is more than 1000 rows
let copy_start = Instant::now();
if !buf.is_empty() {
writer_sink.send(buf.split().freeze()).await?;
}
writer_sink.as_mut().finish().await?;
transaction.batch_execute(&update_sql).await?;
transaction.commit().await?;
transaction = client.transaction().await?;

let duration = copy_start.elapsed().as_millis();
logger.debug(&format!("Copied {collected_row_cnt} rows in {duration}ms"));

writer_sink = Box::pin(
transaction
.copy_in(&format!("COPY {temp_table_name} FROM stdin"))
Expand Down Expand Up @@ -546,12 +551,15 @@ async fn db_exporter_worker(
return Ok(processed_row_cnt);
}

let copy_start = Instant::now();
if !buf.is_empty() {
writer_sink.send(buf.split().freeze()).await?
}
writer_sink.as_mut().finish().await?;
transaction.batch_execute(&update_sql).await?;
transaction.commit().await?;
let duration = copy_start.elapsed().as_millis();
logger.debug(&format!("Copied {collected_row_cnt} rows in {duration}ms"));
logger.info(&format!(
"Embeddings exported to table {} under column {}",
&table, &column
Expand Down

0 comments on commit 88ad71f

Please sign in to comment.