From 094afc1043f31857880722ca8797f5896f3d7c3f Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 29 Apr 2024 12:27:12 +0200 Subject: [PATCH 01/17] add basic tests for the supported types --- Makefile | 3 +- expected/type_support.out | 95 +++++++++++++++++++++++++++++++++++++++ sql/type_support.sql | 47 +++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 expected/type_support.out create mode 100644 sql/type_support.sql diff --git a/Makefile b/Makefile index 72aee371..f881f45d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ SRCS = src/quack_heap_seq_scan.cpp \ OBJS = $(subst .cpp,.o, $(SRCS)) -REGRESS = basic +REGRESS = basic \ + type_support PG_CONFIG ?= pg_config diff --git a/expected/type_support.out b/expected/type_support.out new file mode 100644 index 00000000..e95e2e58 --- /dev/null +++ b/expected/type_support.out @@ -0,0 +1,95 @@ +-- CHAR +CREATE TABLE chr(a CHAR); +INSERT INTO chr SELECT CAST(a AS CHAR) from (VALUES (-128), (0), (127)) t(a); +SELECT * FROM chr; + a +--- + - + 0 + 1 +(3 rows) + +-- SMALLINT +CREATE TABLE small(a SMALLINT); +INSERT INTO small SELECT CAST(a AS SMALLINT) from (VALUES (-32768), (0), (32767)) t(a); +SELECT * FROM small; + a +-------- + -32768 + 0 + 32767 +(3 rows) + +-- INTEGER +CREATE TABLE intgr(a INTEGER); +INSERT INTO intgr SELECT CAST(a AS INTEGER) from (VALUES (-2147483648), (0), (2147483647)) t(a); +SELECT * FROM intgr; + a +------------- + -2147483648 + 0 + 2147483647 +(3 rows) + +-- BIGINT +CREATE TABLE big(a BIGINT); +INSERT INTO big SELECT CAST(a AS BIGINT) from (VALUES (-9223372036854775808), (0), (9223372036854775807)) t(a); +SELECT * FROM big; + a +---------------------- + -9223372036854775808 + 0 + 9223372036854775807 +(3 rows) + +--- BOOL +CREATE TABLE bool_tbl(a BOOL); +INSERT INTO bool_tbl SELECT CAST(a AS BOOL) from (VALUES (False), (NULL), (True)) t(a); +SELECT * FROM bool_tbl; + a +--- + f + + t +(3 rows) + +--- VARCHAR +CREATE TABLE varchar_tbl(a VARCHAR); +INSERT INTO varchar_tbl SELECT CAST(a AS VARCHAR) from (VALUES (''), (NULL), ('test'), ('this is a long string')) t(a); +SELECT * FROM varchar_tbl; + a +----------------------- + + + test + this is a long string +(4 rows) + +-- DATE +CREATE TABLE date_tbl(a DATE); +INSERT INTO date_tbl SELECT CAST(a AS DATE) FROM (VALUES ('2022-04-29'::DATE), (NULL), ('2023-05-15'::DATE)) t(a); +SELECT * FROM date_tbl; + a +------------ + 04-29-2022 + + 05-15-2023 +(3 rows) + +-- TIMESTAMP +CREATE TABLE timestamp_tbl(a TIMESTAMP); +INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); +SELECT * FROM timestamp_tbl; + a +-------------------------- + Fri Apr 29 10:15:30 2022 + + Mon May 15 12:30:45 2023 +(3 rows) + +DROP TABLE small; +DROP TABLE intgr; +DROP TABLE big; +DROP TABLE varchar_tbl; +DROP TABLE date_tbl; +DROP TABLE timestamp_tbl; diff --git a/sql/type_support.sql b/sql/type_support.sql new file mode 100644 index 00000000..7de888c7 --- /dev/null +++ b/sql/type_support.sql @@ -0,0 +1,47 @@ + +-- CHAR +CREATE TABLE chr(a CHAR); +INSERT INTO chr SELECT CAST(a AS CHAR) from (VALUES (-128), (0), (127)) t(a); +SELECT * FROM chr; + +-- SMALLINT +CREATE TABLE small(a SMALLINT); +INSERT INTO small SELECT CAST(a AS SMALLINT) from (VALUES (-32768), (0), (32767)) t(a); +SELECT * FROM small; + +-- INTEGER +CREATE TABLE intgr(a INTEGER); +INSERT INTO intgr SELECT CAST(a AS INTEGER) from (VALUES (-2147483648), (0), (2147483647)) t(a); +SELECT * FROM intgr; + +-- BIGINT +CREATE TABLE big(a BIGINT); +INSERT INTO big SELECT CAST(a AS BIGINT) from (VALUES (-9223372036854775808), (0), (9223372036854775807)) t(a); +SELECT * FROM big; + +--- BOOL +CREATE TABLE bool_tbl(a BOOL); +INSERT INTO bool_tbl SELECT CAST(a AS BOOL) from (VALUES (False), (NULL), (True)) t(a); +SELECT * FROM bool_tbl; + +--- VARCHAR +CREATE TABLE varchar_tbl(a VARCHAR); +INSERT INTO varchar_tbl SELECT CAST(a AS VARCHAR) from (VALUES (''), (NULL), ('test'), ('this is a long string')) t(a); +SELECT * FROM varchar_tbl; + +-- DATE +CREATE TABLE date_tbl(a DATE); +INSERT INTO date_tbl SELECT CAST(a AS DATE) FROM (VALUES ('2022-04-29'::DATE), (NULL), ('2023-05-15'::DATE)) t(a); +SELECT * FROM date_tbl; + +-- TIMESTAMP +CREATE TABLE timestamp_tbl(a TIMESTAMP); +INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); +SELECT * FROM timestamp_tbl; + +DROP TABLE small; +DROP TABLE intgr; +DROP TABLE big; +DROP TABLE varchar_tbl; +DROP TABLE date_tbl; +DROP TABLE timestamp_tbl; From 3fb6598b823b2e569a86f22a0c7a30a46158c88d Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 29 Apr 2024 17:01:09 +0200 Subject: [PATCH 02/17] missing drop for chr --- sql/type_support.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/type_support.sql b/sql/type_support.sql index 7de888c7..159094a5 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -39,6 +39,7 @@ CREATE TABLE timestamp_tbl(a TIMESTAMP); INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); SELECT * FROM timestamp_tbl; +DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; DROP TABLE big; From 8bf0eba1643b4e87aafa39ab6d81122646673676 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 6 May 2024 13:03:45 +0200 Subject: [PATCH 03/17] missing drop echo --- expected/type_support.out | 1 + 1 file changed, 1 insertion(+) diff --git a/expected/type_support.out b/expected/type_support.out index e95e2e58..c5a354dd 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -87,6 +87,7 @@ SELECT * FROM timestamp_tbl; Mon May 15 12:30:45 2023 (3 rows) +DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; DROP TABLE big; From fcec38eb496ba2292608e348cce737aca188c891 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 6 May 2024 16:29:09 +0200 Subject: [PATCH 04/17] numerics without bounds get converted to double, properly detect and propagate this information --- include/quack/quack_types.hpp | 2 +- src/quack_heap_scan.cpp | 5 ++-- src/quack_types.cpp | 50 +++++++++++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/include/quack/quack_types.hpp b/include/quack/quack_types.hpp index 05f38603..62236313 100644 --- a/include/quack/quack_types.hpp +++ b/include/quack/quack_types.hpp @@ -8,7 +8,7 @@ extern "C" { } namespace quack { -duckdb::LogicalType ConvertPostgresToDuckColumnType(Oid type); +duckdb::LogicalType ConvertPostgresToDuckColumnType(Oid type, int32_t typmod); void ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset); void ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col); void InsertTupleIntoChunk(duckdb::DataChunk &output, TupleDesc tuple, HeapTupleData *slot, idx_t offset); diff --git a/src/quack_heap_scan.cpp b/src/quack_heap_scan.cpp index 09d06065..65a13cf2 100644 --- a/src/quack_heap_scan.cpp +++ b/src/quack_heap_scan.cpp @@ -30,7 +30,7 @@ PostgresHeapScanFunctionData::~PostgresHeapScanFunctionData() { PostgresHeapScanGlobalState::PostgresHeapScanGlobalState(PostgresHeapSeqScan &relation) { relation.InitParallelScanState(); - elog(DEBUG3, "-- (DuckDB/PostgresHeapScanGlobalState) Running %lu threads -- ", MaxThreads()); + elog(DEBUG3, "-- (DuckDB/PostgresHeapScanGlobalState) Running %llu threads -- ", MaxThreads()); } PostgresHeapScanGlobalState::~PostgresHeapScanGlobalState() { @@ -79,8 +79,9 @@ PostgresHeapScanFunction::PostgresHeapBind(duckdb::ClientContext &context, duckd for (int i = 0; i < tupleDesc->natts; i++) { Form_pg_attribute attr = &tupleDesc->attrs[i]; Oid type_oid = attr->atttypid; + auto typmod = attr->atttypmod; auto col_name = duckdb::string(NameStr(attr->attname)); - auto duck_type = ConvertPostgresToDuckColumnType(type_oid); + auto duck_type = ConvertPostgresToDuckColumnType(type_oid, typmod); return_types.push_back(duck_type); names.push_back(col_name); /* Log column name and type */ diff --git a/src/quack_types.cpp b/src/quack_types.cpp index 11c9ac34..b80c93ca 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -1,4 +1,5 @@ #include "duckdb.hpp" +#include "duckdb/common/extra_type_info.hpp" extern "C" { #include "postgres.h" @@ -71,8 +72,26 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col } } +static inline int +numeric_typmod_precision(int32 typmod) +{ + return ((typmod - VARHDRSZ) >> 16) & 0xffff; +} + +static inline int +numeric_typmod_scale(int32 typmod) +{ + return (((typmod - VARHDRSZ) & 0x7ff) ^ 1024) - 1024; +} + +struct NumericAsDouble : public duckdb::ExtraTypeInfo { +// Dummy struct to indicate at conversion that the source is a Numeric +public: + NumericAsDouble() : ExtraTypeInfo(duckdb::ExtraTypeInfoType::INVALID_TYPE_INFO) {} +}; + duckdb::LogicalType -ConvertPostgresToDuckColumnType(Oid type) { +ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { switch (type) { case BOOLOID: return duckdb::LogicalTypeId::BOOLEAN; @@ -92,8 +111,19 @@ ConvertPostgresToDuckColumnType(Oid type) { return duckdb::LogicalTypeId::DATE; case TIMESTAMPOID: return duckdb::LogicalTypeId::TIMESTAMP; + case FLOAT8OID: + return duckdb::LogicalTypeId::DOUBLE; + case NUMERICOID: { + if (typmod == -1) { + auto extra_type_info = duckdb::make_shared(); + return duckdb::LogicalType(duckdb::LogicalTypeId::DOUBLE, std::move(extra_type_info)); + } + auto precision = numeric_typmod_precision(typmod); + auto scale = numeric_typmod_scale(typmod); + return duckdb::LogicalType::DECIMAL(precision, scale); + } default: - elog(ERROR, "Unsupported quack type: %d", type); + elog(ERROR, "Unsupported quack (Postgres) type: %d", type); } } @@ -116,8 +146,8 @@ AppendString(duckdb::Vector &result, Datum value, idx_t offset) { void ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { - - switch (result.GetType().id()) { + auto &type = result.GetType(); + switch (type.id()) { case duckdb::LogicalTypeId::BOOLEAN: Append(result, DatumGetBool(value), offset); break; @@ -143,8 +173,18 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { Append(result, duckdb::dtime_t(static_cast(value + QUACK_DUCK_TIMESTAMP_OFFSET)), offset); break; + case duckdb::LogicalTypeId::DOUBLE: { + auto aux_info = type.GetAuxInfoShrPtr(); + if (aux_info && dynamic_cast(aux_info.get())) { + elog(ERROR, "NUMERIC AS DOUBLE"); + } + Append(result, DatumGetFloat8(value), offset); + break; + } + case duckdb::LogicalTypeId::DECIMAL: + elog(ERROR, "DECIMAL TYPE"); default: - elog(ERROR, "Unsupported quack type: %d", static_cast(result.GetType().id())); + elog(ERROR, "Unsupported quack (DuckDB) type: %d", static_cast(result.GetType().id())); break; } } From 0bbc240abc28a4144512323e3fb04e2d2ed49610 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 6 May 2024 17:48:28 +0200 Subject: [PATCH 05/17] convert NUMERIC to DOUBLE --- include/quack/types/decimal.hpp | 273 ++++++++++++++++++++++++++++++++ src/quack_types.cpp | 98 ++++++++++-- 2 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 include/quack/types/decimal.hpp diff --git a/include/quack/types/decimal.hpp b/include/quack/types/decimal.hpp new file mode 100644 index 00000000..ca53a78b --- /dev/null +++ b/include/quack/types/decimal.hpp @@ -0,0 +1,273 @@ +#pragma once + +#define NUMERIC_POS 0x0000 +#define NUMERIC_NEG 0x4000 +#define NUMERIC_NAN 0xC000 +#define NUMERIC_NULL 0xF000 +#define NUMERIC_MAX_PRECISION 1000 +#define NUMERIC_MAX_DISPLAY_SCALE NUMERIC_MAX_PRECISION +#define NUMERIC_MIN_DISPLAY_SCALE 0 +#define NUMERIC_MIN_SIG_DIGITS 16 + +#define NBASE 10000 +#define HALF_NBASE 5000 +#define DEC_DIGITS 4 /* decimal digits per NBASE digit */ +#define MUL_GUARD_DIGITS 2 /* these are measured in NBASE digits */ +#define DIV_GUARD_DIGITS 4 + +#define NUMERIC_EXT_FLAGBITS(n) ((n)->choice.n_header & NUMERIC_EXT_SIGN_MASK) +#define NUMERIC_IS_NAN(n) ((n)->choice.n_header == NUMERIC_NAN) +#define NUMERIC_IS_PINF(n) ((n)->choice.n_header == NUMERIC_PINF) +#define NUMERIC_IS_NINF(n) ((n)->choice.n_header == NUMERIC_NINF) +#define NUMERIC_IS_INF(n) \ + (((n)->choice.n_header & ~NUMERIC_INF_SIGN_MASK) == NUMERIC_PINF) + +/* + * Interpretation of high bits. + */ + +#define NUMERIC_SIGN_MASK 0xC000 +#define NUMERIC_POS 0x0000 +#define NUMERIC_NEG 0x4000 +#define NUMERIC_SHORT 0x8000 +#define NUMERIC_SPECIAL 0xC000 + +#define NUMERIC_FLAGBITS(n) ((n)->choice.n_header & NUMERIC_SIGN_MASK) +#define NUMERIC_IS_SHORT(n) (NUMERIC_FLAGBITS(n) == NUMERIC_SHORT) +#define NUMERIC_IS_SPECIAL(n) (NUMERIC_FLAGBITS(n) == NUMERIC_SPECIAL) + +#define NUMERIC_HDRSZ (VARHDRSZ + sizeof(uint16) + sizeof(int16)) +#define NUMERIC_HDRSZ_SHORT (VARHDRSZ + sizeof(uint16)) + +/* + * If the flag bits are NUMERIC_SHORT or NUMERIC_SPECIAL, we want the short + * header; otherwise, we want the long one. Instead of testing against each + * value, we can just look at the high bit, for a slight efficiency gain. + */ +#define NUMERIC_HEADER_IS_SHORT(n) (((n)->choice.n_header & 0x8000) != 0) +#define NUMERIC_HEADER_SIZE(n) \ + (VARHDRSZ + sizeof(uint16) + \ + (NUMERIC_HEADER_IS_SHORT(n) ? 0 : sizeof(int16))) + +/* + * Definitions for special values (NaN, positive infinity, negative infinity). + * + * The two bits after the NUMERIC_SPECIAL bits are 00 for NaN, 01 for positive + * infinity, 11 for negative infinity. (This makes the sign bit match where + * it is in a short-format value, though we make no use of that at present.) + * We could mask off the remaining bits before testing the active bits, but + * currently those bits must be zeroes, so masking would just add cycles. + */ +#define NUMERIC_EXT_SIGN_MASK 0xF000 /* high bits plus NaN/Inf flag bits */ +#define NUMERIC_NAN 0xC000 +#define NUMERIC_PINF 0xD000 +#define NUMERIC_NINF 0xF000 +#define NUMERIC_INF_SIGN_MASK 0x2000 + +/* + * Short format definitions. + */ + +#define NUMERIC_SHORT_SIGN_MASK 0x2000 +#define NUMERIC_SHORT_DSCALE_MASK 0x1F80 +#define NUMERIC_SHORT_DSCALE_SHIFT 7 +#define NUMERIC_SHORT_DSCALE_MAX \ + (NUMERIC_SHORT_DSCALE_MASK >> NUMERIC_SHORT_DSCALE_SHIFT) +#define NUMERIC_SHORT_WEIGHT_SIGN_MASK 0x0040 +#define NUMERIC_SHORT_WEIGHT_MASK 0x003F +#define NUMERIC_SHORT_WEIGHT_MAX NUMERIC_SHORT_WEIGHT_MASK +#define NUMERIC_SHORT_WEIGHT_MIN (-(NUMERIC_SHORT_WEIGHT_MASK+1)) + +#define NUMERIC_DSCALE_MASK 0x3FFF +#define NUMERIC_DSCALE_MAX NUMERIC_DSCALE_MASK + +#define NUMERIC_SIGN(n) \ + (NUMERIC_IS_SHORT(n) ? \ + (((n)->choice.n_short.n_header & NUMERIC_SHORT_SIGN_MASK) ? \ + NUMERIC_NEG : NUMERIC_POS) : \ + (NUMERIC_IS_SPECIAL(n) ? \ + NUMERIC_EXT_FLAGBITS(n) : NUMERIC_FLAGBITS(n))) +#define NUMERIC_DSCALE(n) (NUMERIC_HEADER_IS_SHORT((n)) ? \ + ((n)->choice.n_short.n_header & NUMERIC_SHORT_DSCALE_MASK) \ + >> NUMERIC_SHORT_DSCALE_SHIFT \ + : ((n)->choice.n_long.n_sign_dscale & NUMERIC_DSCALE_MASK)) +#define NUMERIC_WEIGHT(n) (NUMERIC_HEADER_IS_SHORT((n)) ? \ + (((n)->choice.n_short.n_header & NUMERIC_SHORT_WEIGHT_SIGN_MASK ? \ + ~NUMERIC_SHORT_WEIGHT_MASK : 0) \ + | ((n)->choice.n_short.n_header & NUMERIC_SHORT_WEIGHT_MASK)) \ + : ((n)->choice.n_long.n_weight)) + +#define NUMERIC_DIGITS(num) (NUMERIC_HEADER_IS_SHORT(num) ? \ + (num)->choice.n_short.n_data : (num)->choice.n_long.n_data) +#define NUMERIC_NDIGITS(num) \ + ((VARSIZE(num) - NUMERIC_HEADER_SIZE(num)) / sizeof(NumericDigit)) + +#include "duckdb.hpp" + +extern "C" { +#include "postgres.h" +#include "miscadmin.h" +#include "catalog/pg_type.h" +#include "executor/tuptable.h" +#include "utils/numeric.h" +} + +typedef int16_t NumericDigit; + +struct NumericShort +{ + uint16_t n_header; /* Sign + display scale + weight */ + NumericDigit n_data[FLEXIBLE_ARRAY_MEMBER]; /* Digits */ +}; + +struct NumericLong +{ + uint16_t n_sign_dscale; /* Sign + display scale */ + int16_t n_weight; /* Weight of 1st digit */ + NumericDigit n_data[FLEXIBLE_ARRAY_MEMBER]; /* Digits */ +}; + +union NumericChoice +{ + uint16_t n_header; /* Header word */ + struct NumericLong n_long; /* Long form (4-byte header) */ + struct NumericShort n_short; /* Short form (2-byte header) */ +}; + +struct NumericData +{ + int32_t vl_len_; /* varlena header (do not touch directly!) */ + union NumericChoice choice; /* choice of format */ +}; + +namespace quack { + +struct NumericAsDouble : public duckdb::ExtraTypeInfo { +// Dummy struct to indicate at conversion that the source is a Numeric +public: + NumericAsDouble() : ExtraTypeInfo(duckdb::ExtraTypeInfoType::INVALID_TYPE_INFO) {} +}; + +using duckdb::hugeint_t; + +// Stolen from postgres, they hide these details in numeric.c +typedef struct NumericVar +{ + int32_t ndigits; /* # of digits in digits[] - can be 0! */ + int32_t weight; /* weight of first digit */ + int32_t sign; /* NUMERIC_POS, _NEG, _NAN, _PINF, or _NINF */ + int32_t dscale; /* display scale */ + NumericDigit *buf; /* start of palloc'd space for digits[] */ + NumericDigit *digits; /* base-NBASE digits */ +} NumericVar; + + +NumericVar FromNumeric(Numeric num) +{ + NumericVar dest; + dest.ndigits = NUMERIC_NDIGITS(num); + dest.weight = NUMERIC_WEIGHT(num); + dest.sign = NUMERIC_SIGN(num); + dest.dscale = NUMERIC_DSCALE(num); + dest.digits = NUMERIC_DIGITS(num); + dest.buf = NULL; /* digits array is not palloc'd */ + return dest; +} + +struct DecimalConversionInteger { + static int64_t GetPowerOfTen(idx_t index) { + static const int64_t POWERS_OF_TEN[] {1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000, + 100000000000, + 1000000000000, + 10000000000000, + 100000000000000, + 1000000000000000, + 10000000000000000, + 100000000000000000, + 1000000000000000000}; + if (index >= 19) { + throw duckdb::InternalException("DecimalConversionInteger::GetPowerOfTen - Out of range"); + } + return POWERS_OF_TEN[index]; + } + + template + static T Finalize(const NumericVar &numeric, T result) { + return result; + } +}; + +struct DecimalConversionHugeint { + static hugeint_t GetPowerOfTen(idx_t index) { + static const hugeint_t POWERS_OF_TEN[] { + hugeint_t(1), + hugeint_t(10), + hugeint_t(100), + hugeint_t(1000), + hugeint_t(10000), + hugeint_t(100000), + hugeint_t(1000000), + hugeint_t(10000000), + hugeint_t(100000000), + hugeint_t(1000000000), + hugeint_t(10000000000), + hugeint_t(100000000000), + hugeint_t(1000000000000), + hugeint_t(10000000000000), + hugeint_t(100000000000000), + hugeint_t(1000000000000000), + hugeint_t(10000000000000000), + hugeint_t(100000000000000000), + hugeint_t(1000000000000000000), + hugeint_t(1000000000000000000) * hugeint_t(10), + hugeint_t(1000000000000000000) * hugeint_t(100), + hugeint_t(1000000000000000000) * hugeint_t(1000), + hugeint_t(1000000000000000000) * hugeint_t(10000), + hugeint_t(1000000000000000000) * hugeint_t(100000), + hugeint_t(1000000000000000000) * hugeint_t(1000000), + hugeint_t(1000000000000000000) * hugeint_t(10000000), + hugeint_t(1000000000000000000) * hugeint_t(100000000), + hugeint_t(1000000000000000000) * hugeint_t(1000000000), + hugeint_t(1000000000000000000) * hugeint_t(10000000000), + hugeint_t(1000000000000000000) * hugeint_t(100000000000), + hugeint_t(1000000000000000000) * hugeint_t(1000000000000), + hugeint_t(1000000000000000000) * hugeint_t(10000000000000), + hugeint_t(1000000000000000000) * hugeint_t(100000000000000), + hugeint_t(1000000000000000000) * hugeint_t(1000000000000000), + hugeint_t(1000000000000000000) * hugeint_t(10000000000000000), + hugeint_t(1000000000000000000) * hugeint_t(100000000000000000), + hugeint_t(1000000000000000000) * hugeint_t(1000000000000000000), + hugeint_t(1000000000000000000) * hugeint_t(1000000000000000000) * hugeint_t(10), + hugeint_t(1000000000000000000) * hugeint_t(1000000000000000000) * hugeint_t(100)}; + if (index >= 39) { + throw duckdb::InternalException("DecimalConversionHugeint::GetPowerOfTen - Out of range"); + } + return POWERS_OF_TEN[index]; + } + + static hugeint_t Finalize(const NumericVar &numeric, hugeint_t result) { + return result; + } +}; + +struct DecimalConversionDouble { + static double GetPowerOfTen(idx_t index) { + return pow(10, double(index)); + } + + static double Finalize(const NumericVar &numeric, double result) { + return result / GetPowerOfTen(numeric.dscale); + } +}; + +} // namespace quack diff --git a/src/quack_types.cpp b/src/quack_types.cpp index b80c93ca..8a502da7 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -6,8 +6,10 @@ extern "C" { #include "miscadmin.h" #include "catalog/pg_type.h" #include "executor/tuptable.h" +#include "utils/numeric.h" } +#include "quack/types/decimal.hpp" #include "quack/quack.h" namespace quack { @@ -59,16 +61,20 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col slot->tts_values[col] = timestamp.micros - QUACK_DUCK_TIMESTAMP_OFFSET; break; } - case FLOAT8OID: - case NUMERICOID: { + case FLOAT8OID: { double result_double = value.GetValue(); slot->tts_tupleDescriptor->attrs[col].atttypid = FLOAT8OID; slot->tts_tupleDescriptor->attrs[col].attbyval = true; memcpy(&slot->tts_values[col], (char *)&result_double, sizeof(double)); break; } + case NUMERICOID: { + elog(ERROR, "Unsupported quack (Postgres) type: %d", oid); + break; + } default: - elog(ERROR, "Unsuported quack type: %d", oid); + elog(ERROR, "Unsupported quack (Postgres) type: %d", oid); + break; } } @@ -84,12 +90,6 @@ numeric_typmod_scale(int32 typmod) return (((typmod - VARHDRSZ) & 0x7ff) ^ 1024) - 1024; } -struct NumericAsDouble : public duckdb::ExtraTypeInfo { -// Dummy struct to indicate at conversion that the source is a Numeric -public: - NumericAsDouble() : ExtraTypeInfo(duckdb::ExtraTypeInfoType::INVALID_TYPE_INFO) {} -}; - duckdb::LogicalType ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { switch (type) { @@ -114,12 +114,12 @@ ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { case FLOAT8OID: return duckdb::LogicalTypeId::DOUBLE; case NUMERICOID: { - if (typmod == -1) { + auto precision = numeric_typmod_precision(typmod); + auto scale = numeric_typmod_scale(typmod); + if (typmod == -1 || precision < 0 || scale < 0 || precision > 38) { auto extra_type_info = duckdb::make_shared(); return duckdb::LogicalType(duckdb::LogicalTypeId::DOUBLE, std::move(extra_type_info)); } - auto precision = numeric_typmod_precision(typmod); - auto scale = numeric_typmod_scale(typmod); return duckdb::LogicalType::DECIMAL(precision, scale); } default: @@ -144,6 +144,71 @@ AppendString(duckdb::Vector &result, Datum value, idx_t offset) { data[offset] = duckdb::StringVector::AddString(result, str); } +static bool NumericIsNegative(const NumericVar &numeric) { + return numeric.sign == NUMERIC_NEG; +} + +template +T ConvertDecimal(const NumericVar &numeric) { + auto scale_POWER = OP::GetPowerOfTen(numeric.dscale); + + if (numeric.ndigits == 0) { + return 0; + } + T integral_part = 0, fractional_part = 0; + + if (numeric.weight >= 0) { + idx_t digit_index = 0; + integral_part = numeric.digits[digit_index++]; + for (; digit_index <= numeric.weight; digit_index++) { + integral_part *= NBASE; + if (digit_index < numeric.ndigits) { + integral_part += numeric.digits[digit_index]; + } + } + integral_part *= scale_POWER; + } + + // we need to find out how large the fractional part is in terms of powers + // of ten this depends on how many times we multiplied with NBASE + // if that is different from scale, we need to divide the extra part away + // again + // similarly, if trailing zeroes have been suppressed, we have not been multiplying t + // the fractional part with NBASE often enough. If so, add additional powers + if (numeric.ndigits > numeric.weight + 1) { + auto fractional_power = (numeric.ndigits - numeric.weight - 1) * DEC_DIGITS; + auto fractional_power_correction = fractional_power - numeric.dscale; + D_ASSERT(fractional_power_correction < 20); + fractional_part = 0; + for (int32_t i = duckdb::MaxValue(0, numeric.weight + 1); i < numeric.ndigits; i++) { + if (i + 1 < numeric.ndigits) { + // more digits remain - no need to compensate yet + fractional_part *= NBASE; + fractional_part += numeric.digits[i]; + } else { + // last digit, compensate + T final_base = NBASE; + T final_digit = numeric.digits[i]; + if (fractional_power_correction >= 0) { + T compensation = OP::GetPowerOfTen(fractional_power_correction); + final_base /= compensation; + final_digit /= compensation; + } else { + T compensation = OP::GetPowerOfTen(-fractional_power_correction); + final_base *= compensation; + final_digit *= compensation; + } + fractional_part *= final_base; + fractional_part += final_digit; + } + } + } + + // finally + auto base_res = OP::Finalize(numeric, integral_part + fractional_part); + return (NumericIsNegative(numeric) ? -base_res : base_res); +} + void ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { auto &type = result.GetType(); @@ -176,9 +241,14 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { case duckdb::LogicalTypeId::DOUBLE: { auto aux_info = type.GetAuxInfoShrPtr(); if (aux_info && dynamic_cast(aux_info.get())) { - elog(ERROR, "NUMERIC AS DOUBLE"); + // This NUMERIC could not be converted to a DECIMAL, convert it as DOUBLE instead + auto numeric = DatumGetNumeric(value); + auto numeric_var = FromNumeric(numeric); + auto double_val = ConvertDecimal(numeric_var); + Append(result, double_val, offset); + } else { + Append(result, DatumGetFloat8(value), offset); } - Append(result, DatumGetFloat8(value), offset); break; } case duckdb::LogicalTypeId::DECIMAL: From cedd577395e31ecd1e9d9125fe501ad106d5fb68 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 6 May 2024 17:55:06 +0200 Subject: [PATCH 06/17] detect that the NUMERIC was converted to a DOUBLE, treat it as such --- src/quack_types.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/quack_types.cpp b/src/quack_types.cpp index 8a502da7..ed665f2d 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -18,6 +18,12 @@ namespace quack { constexpr int32_t QUACK_DUCK_DATE_OFFSET = 10957; constexpr int64_t QUACK_DUCK_TIMESTAMP_OFFSET = INT64CONST(10957) * USECS_PER_DAY; +static void ConvertDouble(TupleTableSlot *slot, double value, idx_t col) { + slot->tts_tupleDescriptor->attrs[col].atttypid = FLOAT8OID; + slot->tts_tupleDescriptor->attrs[col].attbyval = true; + memcpy(&slot->tts_values[col], (char *)&value, sizeof(double)); +} + void ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col) { Oid oid = slot->tts_tupleDescriptor->attrs[col].atttypid; @@ -63,12 +69,15 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col } case FLOAT8OID: { double result_double = value.GetValue(); - slot->tts_tupleDescriptor->attrs[col].atttypid = FLOAT8OID; - slot->tts_tupleDescriptor->attrs[col].attbyval = true; - memcpy(&slot->tts_values[col], (char *)&result_double, sizeof(double)); + ConvertDouble(slot, result_double, col); break; } case NUMERICOID: { + if (value.type().id() == duckdb::LogicalTypeId::DOUBLE) { + auto result_double = value.GetValue(); + ConvertDouble(slot, result_double, col); + break; + } elog(ERROR, "Unsupported quack (Postgres) type: %d", oid); break; } From 16767acf0978b1d98bf87198612fc98ac5cb6adc Mon Sep 17 00:00:00 2001 From: Tishj Date: Sun, 12 May 2024 12:56:10 +0200 Subject: [PATCH 07/17] distinguish numeric error message --- src/quack_types.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/quack_types.cpp b/src/quack_types.cpp index ed665f2d..533b4301 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -78,7 +78,7 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col ConvertDouble(slot, result_double, col); break; } - elog(ERROR, "Unsupported quack (Postgres) type: %d", oid); + elog(ERROR, "Unsupported NUMERIC type: %d", oid); break; } default: From b6a79e1466ee1636301784a9aa5000b10b844dc3 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sun, 12 May 2024 13:04:19 +0200 Subject: [PATCH 08/17] add float4 and float8 tests --- expected/type_support.out | 32 ++++++++++++++++++++++++++++++++ sql/type_support.sql | 20 ++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/expected/type_support.out b/expected/type_support.out index c5a354dd..0a20ce51 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -87,6 +87,36 @@ SELECT * FROM timestamp_tbl; Mon May 15 12:30:45 2023 (3 rows) +-- FLOAT4 +CREATE TABLE float4_tbl(a FLOAT4); +INSERT INTO float4_tbl SELECT CAST(a AS FLOAT4) FROM (VALUES + (0.234234234::FLOAT4), + (NULL), + (458234502034234234234.000012::FLOAT4) +) t(a); +SELECT * FROM float4_tbl; + a +-------------- + 0.23423423 + + 4.582345e+20 +(3 rows) + +-- FLOAT8 +CREATE TABLE float8_tbl(a FLOAT8); +INSERT INTO float8_tbl SELECT CAST(a AS FLOAT8) FROM (VALUES + (0.234234234::FLOAT8), + (NULL), + (458234502034234234234.000012::FLOAT8) +) t(a); +SELECT * FROM float8_tbl; + a +----------------------- + 0.234234234 + + 4.582345020342342e+20 +(3 rows) + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -94,3 +124,5 @@ DROP TABLE big; DROP TABLE varchar_tbl; DROP TABLE date_tbl; DROP TABLE timestamp_tbl; +DROP TABLE float4_tbl; +DROP TABLE float8_tbl; diff --git a/sql/type_support.sql b/sql/type_support.sql index 159094a5..d348c31e 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -39,6 +39,24 @@ CREATE TABLE timestamp_tbl(a TIMESTAMP); INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); SELECT * FROM timestamp_tbl; +-- FLOAT4 +CREATE TABLE float4_tbl(a FLOAT4); +INSERT INTO float4_tbl SELECT CAST(a AS FLOAT4) FROM (VALUES + (0.234234234::FLOAT4), + (NULL), + (458234502034234234234.000012::FLOAT4) +) t(a); +SELECT * FROM float4_tbl; + +-- FLOAT8 +CREATE TABLE float8_tbl(a FLOAT8); +INSERT INTO float8_tbl SELECT CAST(a AS FLOAT8) FROM (VALUES + (0.234234234::FLOAT8), + (NULL), + (458234502034234234234.000012::FLOAT8) +) t(a); +SELECT * FROM float8_tbl; + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -46,3 +64,5 @@ DROP TABLE big; DROP TABLE varchar_tbl; DROP TABLE date_tbl; DROP TABLE timestamp_tbl; +DROP TABLE float4_tbl; +DROP TABLE float8_tbl; From 50ba4f8530f9f55a32902c7d41e0b561d456aede Mon Sep 17 00:00:00 2001 From: Tishj Date: Sun, 12 May 2024 13:09:10 +0200 Subject: [PATCH 09/17] add tests for a NUMERIC column that is converted as a DOUBLE because no typmod is present --- expected/type_support.out | 24 ++++++++++++++++++++++++ sql/type_support.sql | 11 +++++++++++ 2 files changed, 35 insertions(+) diff --git a/expected/type_support.out b/expected/type_support.out index 0a20ce51..ebf08f7f 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -117,6 +117,29 @@ SELECT * FROM float8_tbl; 4.582345020342342e+20 (3 rows) +-- NUMERIC as DOUBLE +CREATE TABLE numeric_as_double(a NUMERIC); +INSERT INTO numeric_as_double SELECT a FROM (VALUES + (0.234234234), + (NULL), + (458234502034234234234.000012) +) t(a); +select pg_typeof(a) from numeric_as_double; + pg_typeof +----------- + numeric + numeric + numeric +(3 rows) + +SELECT * FROM numeric_as_double; + a +------------------------------ + 0.234234234 + + 458234502034234234234.000012 +(3 rows) + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -126,3 +149,4 @@ DROP TABLE date_tbl; DROP TABLE timestamp_tbl; DROP TABLE float4_tbl; DROP TABLE float8_tbl; +DROP TABLE numeric_as_double; diff --git a/sql/type_support.sql b/sql/type_support.sql index d348c31e..719f44d3 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -57,6 +57,16 @@ INSERT INTO float8_tbl SELECT CAST(a AS FLOAT8) FROM (VALUES ) t(a); SELECT * FROM float8_tbl; +-- NUMERIC as DOUBLE +CREATE TABLE numeric_as_double(a NUMERIC); +INSERT INTO numeric_as_double SELECT a FROM (VALUES + (0.234234234), + (NULL), + (458234502034234234234.000012) +) t(a); +select pg_typeof(a) from numeric_as_double; +SELECT * FROM numeric_as_double; + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -66,3 +76,4 @@ DROP TABLE date_tbl; DROP TABLE timestamp_tbl; DROP TABLE float4_tbl; DROP TABLE float8_tbl; +DROP TABLE numeric_as_double; From 5e7513ff4b7b6be963a81d92fa7f81375861f3d1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 13 May 2024 16:56:35 +0200 Subject: [PATCH 10/17] make sure the quack extension is loaded in the type test, add support for DECIMALs --- include/quack/types/decimal.hpp | 111 ++++++++++++++++++++++++ sql/type_support.sql | 50 ++++++++++- src/quack_types.cpp | 147 +++++++++++++++++++++++++++++++- 3 files changed, 303 insertions(+), 5 deletions(-) diff --git a/include/quack/types/decimal.hpp b/include/quack/types/decimal.hpp index ca53a78b..0c3c0cfa 100644 --- a/include/quack/types/decimal.hpp +++ b/include/quack/types/decimal.hpp @@ -101,6 +101,10 @@ (num)->choice.n_short.n_data : (num)->choice.n_long.n_data) #define NUMERIC_NDIGITS(num) \ ((VARSIZE(num) - NUMERIC_HEADER_SIZE(num)) / sizeof(NumericDigit)) +#define NUMERIC_CAN_BE_SHORT(scale,weight) \ + ((scale) <= NUMERIC_SHORT_DSCALE_MAX && \ + (weight) <= NUMERIC_SHORT_WEIGHT_MAX && \ + (weight) >= NUMERIC_SHORT_WEIGHT_MIN) #include "duckdb.hpp" @@ -174,6 +178,113 @@ NumericVar FromNumeric(Numeric num) return dest; } +/* + * make_result_opt_error() - + * + * Create the packed db numeric format in palloc()'d memory from + * a variable. This will handle NaN and Infinity cases. + * + * If "have_error" isn't NULL, on overflow *have_error is set to true and + * NULL is returned. This is helpful when caller needs to handle errors. + */ +Numeric CreateNumeric(const NumericVar &var, bool *have_error) +{ + Numeric result; + NumericDigit *digits = var.digits; + int weight = var.weight; + int sign = var.sign; + int n; + Size len; + + if (have_error) + *have_error = false; + + if ((sign & NUMERIC_SIGN_MASK) == NUMERIC_SPECIAL) + { + /* + * Verify valid special value. This could be just an Assert, perhaps, + * but it seems worthwhile to expend a few cycles to ensure that we + * never write any nonzero reserved bits to disk. + */ + if (!(sign == NUMERIC_NAN || + sign == NUMERIC_PINF || + sign == NUMERIC_NINF)) + elog(ERROR, "invalid numeric sign value 0x%x", sign); + + result = (Numeric) palloc(NUMERIC_HDRSZ_SHORT); + + SET_VARSIZE(result, NUMERIC_HDRSZ_SHORT); + result->choice.n_header = sign; + /* the header word is all we need */ + return result; + } + + n = var.ndigits; + + /* truncate leading zeroes */ + while (n > 0 && *digits == 0) + { + digits++; + weight--; + n--; + } + /* truncate trailing zeroes */ + while (n > 0 && digits[n - 1] == 0) + n--; + + /* If zero result, force to weight=0 and positive sign */ + if (n == 0) + { + weight = 0; + sign = NUMERIC_POS; + } + + /* Build the result */ + if (NUMERIC_CAN_BE_SHORT(var.dscale, weight)) + { + len = NUMERIC_HDRSZ_SHORT + n * sizeof(NumericDigit); + result = (Numeric) palloc(len); + SET_VARSIZE(result, len); + result->choice.n_short.n_header = + (sign == NUMERIC_NEG ? (NUMERIC_SHORT | NUMERIC_SHORT_SIGN_MASK) + : NUMERIC_SHORT) + | (var.dscale << NUMERIC_SHORT_DSCALE_SHIFT) + | (weight < 0 ? NUMERIC_SHORT_WEIGHT_SIGN_MASK : 0) + | (weight & NUMERIC_SHORT_WEIGHT_MASK); + } + else + { + len = NUMERIC_HDRSZ + n * sizeof(NumericDigit); + result = (Numeric) palloc(len); + SET_VARSIZE(result, len); + result->choice.n_long.n_sign_dscale = + sign | (var.dscale & NUMERIC_DSCALE_MASK); + result->choice.n_long.n_weight = weight; + } + + Assert(NUMERIC_NDIGITS(result) == n); + if (n > 0) + memcpy(NUMERIC_DIGITS(result), digits, n * sizeof(NumericDigit)); + + /* Check for overflow of int16 fields */ + if (NUMERIC_WEIGHT(result) != weight || + NUMERIC_DSCALE(result) != var.dscale) + { + if (have_error) + { + *have_error = true; + return NULL; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("value overflows numeric format"))); + } + } + return result; +} + struct DecimalConversionInteger { static int64_t GetPowerOfTen(idx_t index) { static const int64_t POWERS_OF_TEN[] {1, diff --git a/sql/type_support.sql b/sql/type_support.sql index 719f44d3..ce49b19d 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -1,3 +1,5 @@ +drop extension if exists quack; +create extension quack; -- CHAR CREATE TABLE chr(a CHAR); @@ -36,9 +38,14 @@ SELECT * FROM date_tbl; -- TIMESTAMP CREATE TABLE timestamp_tbl(a TIMESTAMP); -INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); +INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES + ('2022-04-29 10:15:30'::TIMESTAMP), + (NULL), + ('2023-05-15 12:30:45'::TIMESTAMP) +) t(a); SELECT * FROM timestamp_tbl; +-- FIXME: currently broken -- FLOAT4 CREATE TABLE float4_tbl(a FLOAT4); INSERT INTO float4_tbl SELECT CAST(a AS FLOAT4) FROM (VALUES @@ -64,9 +71,44 @@ INSERT INTO numeric_as_double SELECT a FROM (VALUES (NULL), (458234502034234234234.000012) ) t(a); -select pg_typeof(a) from numeric_as_double; SELECT * FROM numeric_as_double; +-- NUMERIC with a physical type of SMALLINT +CREATE TABLE smallint_numeric(a NUMERIC(4, 2)); +INSERT INTO smallint_numeric SELECT a FROM (VALUES + (0.23), + (NULL), + (45.12) +) t(a); +SELECT * FROM smallint_numeric; + +-- NUMERIC with a physical type of INTEGER +CREATE TABLE integer_numeric(a NUMERIC(9, 6)); +INSERT INTO integer_numeric SELECT a FROM (VALUES + (243.345035::NUMERIC(9,6)), + (NULL), + (45.000012::NUMERIC(9,6)) +) t(a); +SELECT * FROM integer_numeric; + +-- NUMERIC with a physical type of BIGINT +CREATE TABLE bigint_numeric(a NUMERIC(18, 12)); +INSERT INTO bigint_numeric SELECT a FROM (VALUES + (856324.1111222233334444::NUMERIC(18,12)), + (NULL), + (12.0000000000000001::NUMERIC(18,12)) +) t(a); +SELECT * FROM bigint_numeric; + +-- NUMERIC with a physical type of HUGEINT +CREATE TABLE hugeint_numeric(a NUMERIC(38, 24)); +INSERT INTO hugeint_numeric SELECT a FROM (VALUES + (3294234856324.111122223333444455::NUMERIC(38,24)), + (NULL), + (123456789.000000000000000001::NUMERIC(38,24)) -- FIXME: does not convert correctly currently +) t(a); +SELECT * FROM hugeint_numeric; + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -77,3 +119,7 @@ DROP TABLE timestamp_tbl; DROP TABLE float4_tbl; DROP TABLE float8_tbl; DROP TABLE numeric_as_double; +DROP TABLE smallint_numeric; +DROP TABLE integer_numeric; +DROP TABLE bigint_numeric; +DROP TABLE hugeint_numeric; diff --git a/src/quack_types.cpp b/src/quack_types.cpp index 533b4301..9dd843ef 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -24,6 +24,76 @@ static void ConvertDouble(TupleTableSlot *slot, double value, idx_t col) { memcpy(&slot->tts_values[col], (char *)&value, sizeof(double)); } +template +NumericVar ConvertNumeric(T value, idx_t scale) { + NumericVar result; + auto &sign = result.sign; + result.dscale = scale; + auto &weight = result.weight; + auto &ndigits = result.ndigits; + + constexpr idx_t MAX_DIGITS = sizeof(T) * 4; + if (value < 0) { + value = -value; + sign = NUMERIC_NEG; + } else { + sign = NUMERIC_POS; + } + // divide the decimal into the integer part (before the decimal point) and fractional part (after the point) + T integer_part; + T fractional_part; + if (scale == 0) { + integer_part = value; + fractional_part = 0; + } else { + integer_part = value / OP::GetPowerOfTen(scale); + fractional_part = value % OP::GetPowerOfTen(scale); + } + + uint16_t integral_digits[MAX_DIGITS]; + uint16_t fractional_digits[MAX_DIGITS]; + int32_t integral_ndigits; + + // split the integral part into parts of up to NBASE (4 digits => 0..9999) + integral_ndigits = 0; + while (integer_part > 0) { + integral_digits[integral_ndigits++] = uint16_t(integer_part % T(NBASE)); + integer_part /= T(NBASE); + } + weight = integral_ndigits - 1; + // split the fractional part into parts of up to NBASE (4 digits => 0..9999) + // count the amount of digits required for the fractional part + // note that while it is technically possible to leave out zeros here this adds even more complications + // so we just always write digits for the full "scale", even if not strictly required + int32_t fractional_ndigits = (scale + DEC_DIGITS - 1) / DEC_DIGITS; + // fractional digits are LEFT aligned (for some unknown reason) + // that means if we write ".12" with a scale of 2 we actually need to write "1200", instead of "12" + // this means we need to "correct" the number 12 by multiplying by 100 in this case + // this correction factor is the "number of digits to the next full number" + int32_t correction = fractional_ndigits * DEC_DIGITS - scale; + fractional_part *= OP::GetPowerOfTen(correction); + for (idx_t i = 0; i < fractional_ndigits; i++) { + fractional_digits[i] = uint16_t(fractional_part % NBASE); + fractional_part /= NBASE; + } + + ndigits = integral_ndigits + fractional_ndigits; + + result.buf = (NumericDigit *)palloc(ndigits * sizeof(NumericDigit)); + result.digits = result.buf; + auto &digits = result.digits; + + idx_t digits_idx = 0; + for (idx_t i = integral_ndigits; i > 0; i--) { + digits[digits_idx++] = integral_digits[i - 1]; + } + for (idx_t i = fractional_ndigits; i > 0; i--) { + digits[digits_idx++] = fractional_digits[i - 1]; + } + + return result; +} + void ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col) { Oid oid = slot->tts_tupleDescriptor->attrs[col].atttypid; @@ -67,6 +137,13 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col slot->tts_values[col] = timestamp.micros - QUACK_DUCK_TIMESTAMP_OFFSET; break; } + case FLOAT4OID: { + double result_float = value.GetValue(); + slot->tts_tupleDescriptor->attrs[col].atttypid = FLOAT4OID; + slot->tts_tupleDescriptor->attrs[col].attbyval = true; + memcpy(&slot->tts_values[col], (char *)&result_float, sizeof(float)); + break; + } case FLOAT8OID: { double result_double = value.GetValue(); ConvertDouble(slot, result_double, col); @@ -78,7 +155,39 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col ConvertDouble(slot, result_double, col); break; } - elog(ERROR, "Unsupported NUMERIC type: %d", oid); + NumericVar numeric_var; + D_ASSERT(value.type().id() == duckdb::LogicalTypeId::DECIMAL); + auto physical_type = value.type().InternalType(); + auto scale = duckdb::DecimalType::GetScale(value.type()); + switch (physical_type) { + case duckdb::PhysicalType::INT16: { + elog(INFO, "SMALLINT"); + numeric_var = ConvertNumeric(value.GetValueUnsafe(), scale); + break; + } + case duckdb::PhysicalType::INT32: { + elog(INFO, "INTEGER"); + numeric_var = ConvertNumeric(value.GetValueUnsafe(), scale); + break; + } + case duckdb::PhysicalType::INT64: { + elog(INFO, "BIGINT"); + numeric_var = ConvertNumeric(value.GetValueUnsafe(), scale); + break; + } + case duckdb::PhysicalType::INT128: { + elog(INFO, "HUGEINT"); + numeric_var = ConvertNumeric(value.GetValueUnsafe(), scale); + break; + } + default: { + elog(ERROR, "Unrecognized physical type for DECIMAL value"); + break; + } + } + auto numeric = CreateNumeric(numeric_var, NULL); + auto datum = NumericGetDatum(numeric); + slot->tts_values[col] = datum; break; } default: @@ -120,6 +229,8 @@ ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { return duckdb::LogicalTypeId::DATE; case TIMESTAMPOID: return duckdb::LogicalTypeId::TIMESTAMP; + case FLOAT4OID: + return duckdb::LogicalTypeId::FLOAT; case FLOAT8OID: return duckdb::LogicalTypeId::DOUBLE; case NUMERICOID: { @@ -247,6 +358,10 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { Append(result, duckdb::dtime_t(static_cast(value + QUACK_DUCK_TIMESTAMP_OFFSET)), offset); break; + case duckdb::LogicalTypeId::FLOAT: { + Append(result, DatumGetFloat4(value), offset); + break; + } case duckdb::LogicalTypeId::DOUBLE: { auto aux_info = type.GetAuxInfoShrPtr(); if (aux_info && dynamic_cast(aux_info.get())) { @@ -260,8 +375,34 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { } break; } - case duckdb::LogicalTypeId::DECIMAL: - elog(ERROR, "DECIMAL TYPE"); + case duckdb::LogicalTypeId::DECIMAL: { + auto physical_type = type.InternalType(); + auto numeric = DatumGetNumeric(value); + auto numeric_var = FromNumeric(numeric); + switch (physical_type) { + case duckdb::PhysicalType::INT16: { + Append(result, ConvertDecimal(numeric_var), offset); + break; + } + case duckdb::PhysicalType::INT32: { + Append(result, ConvertDecimal(numeric_var), offset); + break; + } + case duckdb::PhysicalType::INT64: { + Append(result, ConvertDecimal(numeric_var), offset); + break; + } + case duckdb::PhysicalType::INT128: { + Append(result, ConvertDecimal(numeric_var), offset); + break; + } + default: { + elog(ERROR, "Unrecognized physical type for DECIMAL value"); + break; + } + } + break; + } default: elog(ERROR, "Unsupported quack (DuckDB) type: %d", static_cast(result.GetType().id())); break; From fa8ceb20c43212157a2cd228362aceac65206ac4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 17 May 2024 13:39:09 +0200 Subject: [PATCH 11/17] fix FLOAT4OID --- sql/type_support.sql | 1 - src/quack_types.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/type_support.sql b/sql/type_support.sql index 91292e1d..ed535ffd 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -45,7 +45,6 @@ INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ) t(a); SELECT * FROM timestamp_tbl; --- FIXME: currently broken -- FLOAT4 CREATE TABLE float4_tbl(a FLOAT4); INSERT INTO float4_tbl SELECT CAST(a AS FLOAT4) FROM (VALUES diff --git a/src/quack_types.cpp b/src/quack_types.cpp index 4410aa35..df4457dc 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -139,7 +139,7 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col break; } case FLOAT4OID: { - double result_float = value.GetValue(); + auto result_float = value.GetValue(); slot->tts_tupleDescriptor->attrs[col].atttypid = FLOAT4OID; slot->tts_tupleDescriptor->attrs[col].attbyval = true; memcpy(&slot->tts_values[col], (char *)&result_float, sizeof(float)); From 7da0e22f11de78995f80320831eac9e9c0cc75b1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 22 May 2024 11:01:40 +0200 Subject: [PATCH 12/17] fix timestamp conversion, update type_support.out --- expected/type_support.out | 96 +++++++++++++++++++++++++++++++++------ src/quack_types.cpp | 6 +-- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/expected/type_support.out b/expected/type_support.out index ebf08f7f..7cff9956 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -1,3 +1,5 @@ +drop extension if exists quack; +create extension quack; -- CHAR CREATE TABLE chr(a CHAR); INSERT INTO chr SELECT CAST(a AS CHAR) from (VALUES (-128), (0), (127)) t(a); @@ -78,7 +80,11 @@ SELECT * FROM date_tbl; -- TIMESTAMP CREATE TABLE timestamp_tbl(a TIMESTAMP); -INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES ('2022-04-29 10:15:30'::TIMESTAMP), (NULL), ('2023-05-15 12:30:45'::TIMESTAMP)) t(a); +INSERT INTO timestamp_tbl SELECT CAST(a AS TIMESTAMP) FROM (VALUES + ('2022-04-29 10:15:30'::TIMESTAMP), + (NULL), + ('2023-05-15 12:30:45'::TIMESTAMP) +) t(a); SELECT * FROM timestamp_tbl; a -------------------------- @@ -124,20 +130,80 @@ INSERT INTO numeric_as_double SELECT a FROM (VALUES (NULL), (458234502034234234234.000012) ) t(a); -select pg_typeof(a) from numeric_as_double; - pg_typeof ------------ - numeric - numeric - numeric +SELECT * FROM numeric_as_double; + a +----------------------- + 0.234234234 + + 4.582345020342342e+20 (3 rows) -SELECT * FROM numeric_as_double; - a ------------------------------- - 0.234234234 - - 458234502034234234234.000012 +-- NUMERIC with a physical type of SMALLINT +CREATE TABLE smallint_numeric(a NUMERIC(4, 2)); +INSERT INTO smallint_numeric SELECT a FROM (VALUES + (0.23), + (NULL), + (45.12) +) t(a); +SELECT * FROM smallint_numeric; +INFO: SMALLINT +INFO: SMALLINT + a +------- + 0.23 + + 45.12 +(3 rows) + +-- NUMERIC with a physical type of INTEGER +CREATE TABLE integer_numeric(a NUMERIC(9, 6)); +INSERT INTO integer_numeric SELECT a FROM (VALUES + (243.345035::NUMERIC(9,6)), + (NULL), + (45.000012::NUMERIC(9,6)) +) t(a); +SELECT * FROM integer_numeric; +INFO: INTEGER +INFO: INTEGER + a +------------ + 243.345035 + + 45.000012 +(3 rows) + +-- NUMERIC with a physical type of BIGINT +CREATE TABLE bigint_numeric(a NUMERIC(18, 12)); +INSERT INTO bigint_numeric SELECT a FROM (VALUES + (856324.111122223333::NUMERIC(18,12)), + (NULL), + (12.000000000001::NUMERIC(18,12)) +) t(a); +SELECT * FROM bigint_numeric; +INFO: BIGINT +INFO: BIGINT + a +--------------------- + 856324.111122223333 + + 12.000000000001 +(3 rows) + +-- NUMERIC with a physical type of HUGEINT +CREATE TABLE hugeint_numeric(a NUMERIC(38, 24)); +INSERT INTO hugeint_numeric SELECT a FROM (VALUES + (32942348563242.111222333444555666777888::NUMERIC(38,24)), + (NULL), + (123456789.000000000000000000000001::NUMERIC(38,24)) +) t(a); +SELECT * FROM hugeint_numeric; +INFO: HUGEINT +INFO: HUGEINT + a +----------------------------------------- + 32942348563242.111222333444555666777888 + + 123456789.000000000000000000000001 (3 rows) DROP TABLE chr; @@ -150,3 +216,7 @@ DROP TABLE timestamp_tbl; DROP TABLE float4_tbl; DROP TABLE float8_tbl; DROP TABLE numeric_as_double; +DROP TABLE smallint_numeric; +DROP TABLE integer_numeric; +DROP TABLE bigint_numeric; +DROP TABLE hugeint_numeric; diff --git a/src/quack_types.cpp b/src/quack_types.cpp index df4457dc..e899808d 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -134,8 +134,8 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col break; } case TIMESTAMPOID: { - duckdb::dtime_t timestamp = value.GetValue(); - slot->tts_values[col] = timestamp.micros - quack::QUACK_DUCK_TIMESTAMP_OFFSET; + duckdb::timestamp_t timestamp = value.GetValue(); + slot->tts_values[col] = timestamp.value - quack::QUACK_DUCK_TIMESTAMP_OFFSET; break; } case FLOAT4OID: { @@ -388,7 +388,7 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { Append(result, duckdb::date_t(static_cast(value + QUACK_DUCK_DATE_OFFSET)), offset); break; case duckdb::LogicalTypeId::TIMESTAMP: - Append(result, duckdb::dtime_t(static_cast(value + QUACK_DUCK_TIMESTAMP_OFFSET)), + Append(result, duckdb::timestamp_t(static_cast(value + QUACK_DUCK_TIMESTAMP_OFFSET)), offset); break; case duckdb::LogicalTypeId::FLOAT: { From 86352d868cdb191c0d6b451548674eb8f5bda0ed Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 22 May 2024 14:03:29 +0200 Subject: [PATCH 13/17] add UUID type support --- expected/type_support.out | 16 ++++++++++++++++ sql/type_support.sql | 10 ++++++++++ src/quack_types.cpp | 38 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/expected/type_support.out b/expected/type_support.out index 7cff9956..07713c48 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -206,6 +206,21 @@ INFO: HUGEINT 123456789.000000000000000000000001 (3 rows) +-- UUID +CREATE TABLE uuid_tbl(a UUID); +INSERT INTO uuid_tbl SELECT CAST(a as UUID) FROM (VALUES + ('80bf0be9-89be-4ef8-bc58-fc7d691c5544'), + (NULL), + ('00000000-0000-0000-0000-000000000000') +) t(a); +SELECT * FROM uuid_tbl; + a +-------------------------------------- + 80bf0be9-89be-4ef8-bc58-fc7d691c5544 + + 00000000-0000-0000-0000-000000000000 +(3 rows) + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -220,3 +235,4 @@ DROP TABLE smallint_numeric; DROP TABLE integer_numeric; DROP TABLE bigint_numeric; DROP TABLE hugeint_numeric; +DROP TABLE uuid_tbl; diff --git a/sql/type_support.sql b/sql/type_support.sql index ed535ffd..4c6da769 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -108,6 +108,15 @@ INSERT INTO hugeint_numeric SELECT a FROM (VALUES ) t(a); SELECT * FROM hugeint_numeric; +-- UUID +CREATE TABLE uuid_tbl(a UUID); +INSERT INTO uuid_tbl SELECT CAST(a as UUID) FROM (VALUES + ('80bf0be9-89be-4ef8-bc58-fc7d691c5544'), + (NULL), + ('00000000-0000-0000-0000-000000000000') +) t(a); +SELECT * FROM uuid_tbl; + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -122,3 +131,4 @@ DROP TABLE smallint_numeric; DROP TABLE integer_numeric; DROP TABLE bigint_numeric; DROP TABLE hugeint_numeric; +DROP TABLE uuid_tbl; \ No newline at end of file diff --git a/src/quack_types.cpp b/src/quack_types.cpp index e899808d..3831d3e2 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -1,5 +1,6 @@ #include "duckdb.hpp" #include "duckdb/common/extra_type_info.hpp" +#include "duckdb/common/types/uuid.hpp" extern "C" { #include "postgres.h" @@ -8,6 +9,7 @@ extern "C" { #include "catalog/pg_type.h" #include "executor/tuptable.h" #include "utils/numeric.h" +#include "utils/uuid.h" } #include "quack/types/decimal.hpp" @@ -191,6 +193,24 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col slot->tts_values[col] = datum; break; } + case UUIDOID: { + D_ASSERT(value.type().id() == duckdb::LogicalTypeId::UUID); + D_ASSERT(value.type().InternalType() == duckdb::PhysicalType::INT128); + auto duckdb_uuid = value.GetValue(); + pg_uuid_t *postgres_uuid = (pg_uuid_t *) palloc(sizeof(pg_uuid_t)); + + duckdb_uuid.upper ^= (uint64_t(1) << 63); + // Convert duckdb_uuid to bytes and store in postgres_uuid.data + uint8_t *uuid_bytes = (uint8_t *)&duckdb_uuid; + + for (int i = 0; i < UUID_LEN; ++i) { + postgres_uuid->data[i] = uuid_bytes[UUID_LEN - 1 - i]; + } + + auto datum = UUIDPGetDatum(postgres_uuid); + slot->tts_values[col] = datum; + break; + } default: elog(ERROR, "(DuckDB/ConvertDuckToPostgresValue) Unsuported quack type: %d", oid); } @@ -242,6 +262,8 @@ ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { } return duckdb::LogicalType::DECIMAL(precision, scale); } + case UUIDOID: + return duckdb::LogicalTypeId::UUID; default: elog(ERROR, "(DuckDB/ConvertPostgresToDuckColumnType) Unsupported quack type: %d", type); } @@ -275,6 +297,8 @@ GetPostgresDuckDBType(duckdb::LogicalTypeId type) { case duckdb::LogicalTypeId::DECIMAL: { return NUMERICOID; } + case duckdb::LogicalTypeId::UUID: + return UUIDOID; default: elog(ERROR, "(DuckDB/GetPostgresDuckDBType) Unsupported quack type: %d", static_cast(type)); } @@ -436,9 +460,19 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { } break; } + case duckdb::LogicalTypeId::UUID: { + auto uuid = DatumGetPointer(value); + hugeint_t duckdb_uuid; + D_ASSERT(UUID_LEN == sizeof(hugeint_t)); + for (idx_t i = 0; i < UUID_LEN; i++) { + ((uint8_t*)&duckdb_uuid)[UUID_LEN-1-i] = ((uint8_t*)uuid)[i]; + } + duckdb_uuid.upper ^= (uint64_t(1) << 63); + Append(result, duckdb_uuid, offset); + break; + } default: - elog(ERROR, "(DuckDB/ConvertPostgresToDuckValue) Unsupported quack type: %d", - static_cast(result.GetType().id())); + elog(ERROR, "(DuckDB/ConvertPostgresToDuckValue) Unsupported quack type: %s", duckdb::EnumUtil::ToChars(result.GetType().id())); break; } } From dfdf7697a0b47f2823d68a253710fdf74c1afc9f Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 22 May 2024 14:49:48 +0200 Subject: [PATCH 14/17] add JSON type support --- Makefile | 9 ++++++++- include/quack/quack_types.hpp | 2 +- sql/type_support.sql | 13 ++++++++++++- src/quack_planner.cpp | 2 +- src/quack_types.cpp | 24 ++++++++++++++++++------ 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index fba5f04f..f6cc1ff0 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,14 @@ third_party/duckdb/Makefile: git submodule update --init --recursive third_party/duckdb/build/$(QUACK_BUILD_DUCKDB)/src/$(DUCKDB_LIB): - $(MAKE) -C third_party/duckdb $(QUACK_BUILD_DUCKDB) DISABLE_SANITIZER=1 ENABLE_UBSAN=0 BUILD_UNITTESTS=OFF BUILD_HTTPFS=1 CMAKE_EXPORT_COMPILE_COMMANDS=1 + $(MAKE) -C third_party/duckdb \ + $(QUACK_BUILD_DUCKDB) \ + DISABLE_SANITIZER=1 \ + ENABLE_UBSAN=0 \ + BUILD_UNITTESTS=OFF \ + BUILD_HTTPFS=1 \ + BUILD_JSON=1 \ + CMAKE_EXPORT_COMPILE_COMMANDS=1 install_duckdb: $(install_bin) -m 755 third_party/duckdb/build/$(QUACK_BUILD_DUCKDB)/src/$(DUCKDB_LIB) $(DESTDIR)$(PG_LIB) diff --git a/include/quack/quack_types.hpp b/include/quack/quack_types.hpp index 82fdf428..d2ad9f0e 100644 --- a/include/quack/quack_types.hpp +++ b/include/quack/quack_types.hpp @@ -16,7 +16,7 @@ constexpr int32_t QUACK_DUCK_DATE_OFFSET = 10957; constexpr int64_t QUACK_DUCK_TIMESTAMP_OFFSET = INT64CONST(10957) * USECS_PER_DAY; duckdb::LogicalType ConvertPostgresToDuckColumnType(Oid type, int32_t typmod); -Oid GetPostgresDuckDBType(duckdb::LogicalTypeId type); +Oid GetPostgresDuckDBType(duckdb::LogicalType type); void ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset); void ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col); void InsertTupleIntoChunk(duckdb::DataChunk &output, PostgresHeapSeqScanThreadInfo &threadScanInfo, diff --git a/sql/type_support.sql b/sql/type_support.sql index 4c6da769..fe24d491 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -117,6 +117,16 @@ INSERT INTO uuid_tbl SELECT CAST(a as UUID) FROM (VALUES ) t(a); SELECT * FROM uuid_tbl; +-- JSON +CREATE TABLE json_tbl(a JSON); +INSERT INTO json_tbl SELECT CAST(a as JSON) FROM (VALUES + ('{"key1": "value1", "key2": "value2"}'), + ('["item1", "item2", "item3"]'), + (NULL), + ('{}') +) t(a); +SELECT * FROM json_tbl; + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -131,4 +141,5 @@ DROP TABLE smallint_numeric; DROP TABLE integer_numeric; DROP TABLE bigint_numeric; DROP TABLE hugeint_numeric; -DROP TABLE uuid_tbl; \ No newline at end of file +DROP TABLE uuid_tbl; +DROP TABLE json_tbl; diff --git a/src/quack_planner.cpp b/src/quack_planner.cpp index e748ceca..d0e25e71 100644 --- a/src/quack_planner.cpp +++ b/src/quack_planner.cpp @@ -71,7 +71,7 @@ quack_create_plan(Query *parse, const char *query) { for (auto i = 0; i < preparedResultTypes.size(); i++) { auto &column = preparedResultTypes[i]; - Oid postgresColumnOid = quack::GetPostgresDuckDBType(column.id()); + Oid postgresColumnOid = quack::GetPostgresDuckDBType(column); if (OidIsValid(postgresColumnOid)) { HeapTuple tp; diff --git a/src/quack_types.cpp b/src/quack_types.cpp index 3831d3e2..1a0c4902 100644 --- a/src/quack_types.cpp +++ b/src/quack_types.cpp @@ -119,6 +119,7 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col break; case BPCHAROID: case TEXTOID: + case JSONOID: case VARCHAROID: { auto str = value.GetValue(); auto varchar = str.c_str(); @@ -264,14 +265,17 @@ ConvertPostgresToDuckColumnType(Oid type, int32_t typmod) { } case UUIDOID: return duckdb::LogicalTypeId::UUID; + case JSONOID: + return duckdb::LogicalType::JSON(); default: elog(ERROR, "(DuckDB/ConvertPostgresToDuckColumnType) Unsupported quack type: %d", type); } } Oid -GetPostgresDuckDBType(duckdb::LogicalTypeId type) { - switch (type) { +GetPostgresDuckDBType(duckdb::LogicalType type) { + auto id = type.id(); + switch (id) { case duckdb::LogicalTypeId::BOOLEAN: return BOOLOID; case duckdb::LogicalTypeId::TINYINT: @@ -284,8 +288,12 @@ GetPostgresDuckDBType(duckdb::LogicalTypeId type) { return INT8OID; case duckdb::LogicalTypeId::HUGEINT: return NUMERICOID; - case duckdb::LogicalTypeId::VARCHAR: + case duckdb::LogicalTypeId::VARCHAR: { + if (type.IsJSONType()) { + return JSONOID; + } return VARCHAROID; + } case duckdb::LogicalTypeId::DATE: return DATEOID; case duckdb::LogicalTypeId::TIMESTAMP: @@ -299,8 +307,10 @@ GetPostgresDuckDBType(duckdb::LogicalTypeId type) { } case duckdb::LogicalTypeId::UUID: return UUIDOID; - default: - elog(ERROR, "(DuckDB/GetPostgresDuckDBType) Unsupported quack type: %d", static_cast(type)); + default: { + elog(ERROR, "(DuckDB/GetPostgresDuckDBType) Unsupported quack type: %s", type.ToString().c_str()); + break; + } } } @@ -405,9 +415,11 @@ ConvertPostgresToDuckValue(Datum value, duckdb::Vector &result, idx_t offset) { case duckdb::LogicalTypeId::BIGINT: Append(result, DatumGetInt64(value), offset); break; - case duckdb::LogicalTypeId::VARCHAR: + case duckdb::LogicalTypeId::VARCHAR: { + // NOTE: This also handles JSON AppendString(result, value, offset); break; + } case duckdb::LogicalTypeId::DATE: Append(result, duckdb::date_t(static_cast(value + QUACK_DUCK_DATE_OFFSET)), offset); break; From 78d66cf04da47cf4137ca52c35a61881614d1019 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 27 May 2024 16:55:14 +0200 Subject: [PATCH 15/17] fix up tests and compilation (uint64 -> uint64_t) --- expected/type_support.out | 19 ++++++++++++++++++- sql/type_support.sql | 1 - src/utility/copy.cpp | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/expected/type_support.out b/expected/type_support.out index 07713c48..3bff37af 100644 --- a/expected/type_support.out +++ b/expected/type_support.out @@ -1,4 +1,3 @@ -drop extension if exists quack; create extension quack; -- CHAR CREATE TABLE chr(a CHAR); @@ -221,6 +220,23 @@ SELECT * FROM uuid_tbl; 00000000-0000-0000-0000-000000000000 (3 rows) +-- JSON +CREATE TABLE json_tbl(a JSON); +INSERT INTO json_tbl SELECT CAST(a as JSON) FROM (VALUES + ('{"key1": "value1", "key2": "value2"}'), + ('["item1", "item2", "item3"]'), + (NULL), + ('{}') +) t(a); +SELECT * FROM json_tbl; + a +-------------------------------------- + {"key1": "value1", "key2": "value2"} + ["item1", "item2", "item3"] + + {} +(4 rows) + DROP TABLE chr; DROP TABLE small; DROP TABLE intgr; @@ -236,3 +252,4 @@ DROP TABLE integer_numeric; DROP TABLE bigint_numeric; DROP TABLE hugeint_numeric; DROP TABLE uuid_tbl; +DROP TABLE json_tbl; diff --git a/sql/type_support.sql b/sql/type_support.sql index fe24d491..12826bb8 100644 --- a/sql/type_support.sql +++ b/sql/type_support.sql @@ -1,4 +1,3 @@ -drop extension if exists quack; create extension quack; -- CHAR diff --git a/src/utility/copy.cpp b/src/utility/copy.cpp index a50de531..a1af1f71 100644 --- a/src/utility/copy.cpp +++ b/src/utility/copy.cpp @@ -129,6 +129,6 @@ quack_copy(PlannedStmt *pstmt, const char *queryString, struct QueryEnvironment } auto chunk = res->Fetch(); - *processed = chunk->GetValue(0, 0).GetValue(); + *processed = chunk->GetValue(0, 0).GetValue(); return true; } From 32d0d8a094d581fb7b0982735d2f8e7a0ba5cef1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 28 May 2024 08:53:26 +0200 Subject: [PATCH 16/17] missing include --- include/quack/types/decimal.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/quack/types/decimal.hpp b/include/quack/types/decimal.hpp index 0c3c0cfa..77760e23 100644 --- a/include/quack/types/decimal.hpp +++ b/include/quack/types/decimal.hpp @@ -107,6 +107,7 @@ (weight) >= NUMERIC_SHORT_WEIGHT_MIN) #include "duckdb.hpp" +#include extern "C" { #include "postgres.h" From 759d0d8303449f63383d898b4b736ab9854ca4c7 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 29 May 2024 11:04:25 +0200 Subject: [PATCH 17/17] undo changes to makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0d43d37d..5602d0b6 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ check-regression-quack: clean-regression: $(MAKE) -C test/regression clean-regression -installcheck: check-regression-quack +installcheck: all install check-regression-quack duckdb: third_party/duckdb/Makefile third_party/duckdb/build/$(QUACK_BUILD_DUCKDB)/src/$(DUCKDB_LIB)