Detoast columns that are needed (#581)

If columns contains value that could not fit into heap page we need to detoast it before passing to duckdb execution.
duckdb · Feb 7, 2025 · a9faa3c · a9faa3c
1 parent de3735a
commit a9faa3c
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 5 deletions.
diff --git a/src/pgduckdb_detoast.cpp b/src/pgduckdb_detoast.cpp
@@ -126,8 +126,6 @@ ToastFetchDatum(struct varlena *attr) {
 		return result;
 	}
 
-	std::lock_guard<std::mutex> lock(GlobalProcessLock::GetLock());
-
 	if (!PostgresFunctionGuard(table_relation_fetch_toast_slice, toast_pointer, attrsize, result)) {
 		duckdb_free(result);
 		throw duckdb::InternalException("(PGDuckDB/ToastFetchDatum) Error toast relation is NULL");

diff --git a/src/pgduckdb_types.cpp b/src/pgduckdb_types.cpp
@@ -1468,10 +1468,19 @@ InsertTupleIntoChunk(duckdb::DataChunk &output, PostgresScanLocalState &scan_loc
 			auto &array_mask = duckdb::FlatVector::Validity(result);
 			array_mask.SetInvalid(scan_local_state.output_vector_size);
 		} else {
-			/* Use returned tuple slot attr information. */
 			auto attr = slot->tts_tupleDescriptor->attrs[duckdb_output_index];
-			ConvertPostgresToDuckValue(attr.atttypid, slot->tts_values[duckdb_output_index], result,
-			                           scan_local_state.output_vector_size);
+			if (attr.attlen == -1) {
+				bool should_free = false;
+				Datum detoasted_value = DetoastPostgresDatum(
+				    reinterpret_cast<varlena *>(slot->tts_values[duckdb_output_index]), &should_free);
+				ConvertPostgresToDuckValue(attr.atttypid, detoasted_value, result, scan_local_state.output_vector_size);
+				if (should_free) {
+					duckdb_free(reinterpret_cast<void *>(detoasted_value));
+				}
+			} else {
+				ConvertPostgresToDuckValue(attr.atttypid, slot->tts_values[duckdb_output_index], result,
+				                           scan_local_state.output_vector_size);
+			}
 		}
 	}
 

diff --git a/test/regression/expected/basic.out b/test/regression/expected/basic.out
@@ -64,3 +64,13 @@ SELECT * FROM t JOIN s ON a = b;
 
 DROP TABLE t;
 DROP TABLE s;
+-- Check that we are counting detoasted value
+CREATE TABLE t(a INT, b VARCHAR);
+INSERT INTO t SELECT g, repeat('ABCDE', 10000) FROM generate_series(1, 10) g;
+SELECT LENGTH(b) FROM t WHERE a = 5;
+ length 
+--------
+  50000
+(1 row)
+
+DROP TABLE t;
diff --git a/test/regression/sql/basic.sql b/test/regression/sql/basic.sql
@@ -47,3 +47,9 @@ CREATE TABLE s (b INT);
 SELECT * FROM t JOIN s ON a = b;
 DROP TABLE t;
 DROP TABLE s;
+
+-- Check that we are counting detoasted value
+CREATE TABLE t(a INT, b VARCHAR);
+INSERT INTO t SELECT g, repeat('ABCDE', 10000) FROM generate_series(1, 10) g;
+SELECT LENGTH(b) FROM t WHERE a = 5;
+DROP TABLE t;