Use auto increment on non primary indexes

.czrc

@@ -0,0 +1,3 @@
+{
+  "path": "cz-conventional-changelog"
+}

embedded/sql/catalog.go

@@ -29,17 +29,17 @@ type Database struct {
 }
 
 type Table struct {
-	db              *Database
-	id              uint32
-	name            string
-	cols            []*Column
-	colsByID        map[uint32]*Column
-	colsByName      map[string]*Column
-	indexes         map[string]*Index
-	indexesByColID  map[uint32][]*Index
-	primaryIndex    *Index
-	autoIncrementPK bool
-	maxPK           int64
+	db                 *Database
+	id                 uint32
+	name               string
+	cols               []*Column
+	colsByID           map[uint32]*Column
+	colsByName         map[string]*Column
+	indexes            map[string]*Index
+	indexesByColID     map[uint32][]*Index
+	primaryIndex       *Index
+	autoIncrement      *Index
+	maxPK              int64
 }
 
 type Index struct {
@@ -224,6 +224,26 @@ func (t *Table) GetColumnByID(id uint32) (*Column, error) {
 	return col, nil
 }
 
+func (t *Table) IsAutoIncremented() bool {
+	for _, c := range t.cols {
+		if c.autoIncrement {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (t *Table) GetAutoIncrementedColumn() (cols []*Column) {
+	for _, c := range t.cols {
+		if c.autoIncrement {
+			cols = append(cols, c)
+		}
+	}
+
+	return
+}
+
 func (i *Index) IsPrimary() bool {
 	return i.id == PKIndexID
 }
@@ -300,7 +320,7 @@ func (db *Database) newTable(name string, colsSpec []*ColSpec) (table *Table, er
 		}
 
 		if cs.autoIncrement && cs.colType != IntegerType {
-			return nil, ErrLimitedAutoIncrement
+			return nil, ErrAutoIncrementWrongType
 		}
 
 		if !validMaxLenForType(cs.maxLen, cs.colType) {
@@ -374,13 +394,23 @@ func (t *Table) newIndex(unique bool, colIDs []uint32) (index *Index, err error)
 	// having a direct way to get the indexes by colID
 	for _, col := range index.cols {
 		t.indexesByColID[col.id] = append(t.indexesByColID[col.id], index)
+
+		if col.autoIncrement {
+			// cannot technically happen if the table creation, was properly handled. But it's better to handle the error
+			// TODO: support multi column auto_increment
+			if len(t.GetAutoIncrementedColumn()) > 1 {
+				return nil, ErrAutoIncrementMultiple
+			}
+
+			t.autoIncrement = index
+		}
 	}
 
 	if index.id == PKIndexID {
 		t.primaryIndex = index
-		t.autoIncrementPK = len(index.cols) == 1 && index.cols[0].autoIncrement
 	}
 
+
 	return index, nil
 }
 

embedded/sql/engine.go

@@ -39,7 +39,9 @@ var ErrTableDoesNotExist = errors.New("table does not exist")
 var ErrColumnDoesNotExist = errors.New("column does not exist")
 var ErrColumnNotIndexed = errors.New("column is not indexed")
 var ErrLimitedKeyType = errors.New("indexed key of invalid type. Supported types are: INTEGER, VARCHAR[256] OR BLOB[256]")
-var ErrLimitedAutoIncrement = errors.New("only INTEGER single-column primary keys can be set as auto incremental")
+var ErrTableNotAutoIncremented = errors.New("the table has no auto incremented column")
+var ErrAutoIncrementWrongType = errors.New("auto incremented column need to be INTEGER type")
+var ErrAutoIncrementMultiple = errors.New("several auto incremental column were found. Wrong schema")
 var ErrNoValueForAutoIncrementalColumn = errors.New("no value should be specified for auto incremental columns")
 var ErrLimitedMaxLen = errors.New("only VARCHAR and BLOB types support max length")
 var ErrDuplicatedColumn = errors.New("duplicated column")
@@ -387,7 +389,7 @@ func (db *Database) loadTables(sqlPrefix []byte, tx *store.OngoingTx) error {
 			return err
 		}
 
-		if table.autoIncrementPK {
+		if table.IsAutoIncremented()  {
 			encMaxPK, err := loadMaxPK(sqlPrefix, tx, table)
 			if err == store.ErrNoMoreEntries {
 				continue
@@ -426,7 +428,7 @@ func indexKeyFrom(cols []*Column) string {
 
 func loadMaxPK(sqlPrefix []byte, tx *store.OngoingTx, table *Table) ([]byte, error) {
 	pkReaderSpec := &store.KeyReaderSpec{
-		Prefix:    mapKey(sqlPrefix, PIndexPrefix, EncodeID(table.db.id), EncodeID(table.id), EncodeID(PKIndexID)),
+		Prefix:    mapKey(sqlPrefix, table.autoIncrement.prefix(), EncodeID(table.db.id), EncodeID(table.id), EncodeID(table.autoIncrement.id)),
 		DescOrder: true,
 	}
 
@@ -441,7 +443,7 @@ func loadMaxPK(sqlPrefix []byte, tx *store.OngoingTx, table *Table) ([]byte, err
 		return nil, err
 	}
 
-	return unmapIndexEntry(table.primaryIndex, sqlPrefix, mkey)
+	return unmapIndexEntry(table.autoIncrement, sqlPrefix, mkey)
 }
 
 func loadColSpecs(dbID, tableID uint32, tx *store.OngoingTx, sqlPrefix []byte) (specs []*ColSpec, err error) {
@@ -693,7 +695,7 @@ func unmapIndexEntry(index *Index, sqlPrefix, mkey []byte) (encPKVals []byte, er
 		return nil, ErrCorruptedData
 	}
 
-	if !index.IsPrimary() {
+	if !index.IsPrimary() && !index.table.IsAutoIncremented() {
 		//read index values
 		for _, col := range index.cols {
 			if enc[off] == KeyValPrefixNull {

embedded/sql/engine_test.go

@@ -597,6 +597,18 @@ func TestInsertIntoEdgeCases(t *testing.T) {
 	_, _, err = engine.Exec("INSERT INTO table1 (id, title, active, payload) VALUES (1, 'title1', true, x'00A1')", nil, nil)
 	require.NoError(t, err)
 
+	t.Run("on conflict cases", func(t *testing.T) {
+		_, _, err = engine.Exec("INSERT INTO table1 (id, title, active, payload) VALUES (1, 'title1', true, x'00A1')", nil, nil)
+		require.ErrorIs(t, err, store.ErrKeyAlreadyExists)
+
+		ntx, ctxs, err := engine.Exec("INSERT INTO table1 (id, title, active, payload) VALUES (1, 'title1', true, x'00A1') ON CONFLICT DO NOTHING", nil, nil)
+		require.NoError(t, err)
+		require.Nil(t, ntx)
+		require.Len(t, ctxs, 1)
+		require.Zero(t, ctxs[0].UpdatedRows())
+		require.Nil(t, ctxs[0].TxHeader())
+	})
+
 	t.Run("varchar key cases", func(t *testing.T) {
 		_, _, err = engine.Exec("INSERT INTO table1 (id, title, active, payload) VALUES (2, 'title123456789', true, x'00A1')", nil, nil)
 		require.ErrorIs(t, err, ErrMaxLengthExceeded)
@@ -617,6 +629,14 @@ func TestInsertIntoEdgeCases(t *testing.T) {
 		_, _, err = engine.Exec("INSERT INTO table1 (id, title, active, payload) VALUES (2, 'title1', true, '00A100A2')", nil, nil)
 		require.ErrorIs(t, err, ErrInvalidValue)
 	})
+
+	t.Run("insertion in table with varchar pk", func(t *testing.T) {
+		_, _, err = engine.Exec("CREATE TABLE languages (code VARCHAR[255],name VARCHAR[255],PRIMARY KEY code)", nil, nil)
+		require.NoError(t, err)
+
+		_, _, err = engine.Exec("INSERT INTO languages (code,name) VALUES ('code1', 'name1')", nil, nil)
+		require.NoError(t, err)
+	})
 }
 
 func TestAutoIncrementPK(t *testing.T) {
@@ -633,18 +653,12 @@ func TestAutoIncrementPK(t *testing.T) {
 	err = engine.SetDefaultDatabase("db1")
 	require.NoError(t, err)
 
-	t.Run("invalid use of auto-increment", func(t *testing.T) {
-		_, _, err = engine.Exec("CREATE TABLE table1 (id INTEGER, title VARCHAR AUTO_INCREMENT, PRIMARY KEY id)", nil, nil)
-		require.ErrorIs(t, err, ErrLimitedAutoIncrement)
-
-		_, _, err = engine.Exec("CREATE TABLE table1 (id INTEGER, title VARCHAR, age INTEGER AUTO_INCREMENT, PRIMARY KEY id)", nil, nil)
-		require.ErrorIs(t, err, ErrLimitedAutoIncrement)
-
+	t.Run("wrong auto-increment type", func(t *testing.T) {
 		_, _, err = engine.Exec("CREATE TABLE table1 (id VARCHAR AUTO_INCREMENT, title VARCHAR, PRIMARY KEY id)", nil, nil)
-		require.ErrorIs(t, err, ErrLimitedAutoIncrement)
+		require.ErrorIs(t, err, ErrAutoIncrementWrongType)
 	})
 
-	_, _, err = engine.Exec("CREATE TABLE table1 (id INTEGER AUTO_INCREMENT, title VARCHAR, PRIMARY KEY id)", nil, nil)
+	_, _, err = engine.Exec("CREATE TABLE table1 (id INTEGER NOT NULL AUTO_INCREMENT, title VARCHAR, PRIMARY KEY id)", nil, nil)
 	require.NoError(t, err)
 
 	_, ctxs, err := engine.Exec("INSERT INTO table1(title) VALUES ('name1')", nil, nil)
@@ -654,26 +668,88 @@ func TestAutoIncrementPK(t *testing.T) {
 	require.Equal(t, int64(1), ctxs[0].LastInsertedPKs()["table1"])
 	require.Equal(t, 1, ctxs[0].UpdatedRows())
 
-	_, _, err = engine.Exec("INSERT INTO table1(id, title) VALUES (2, 'name2')", nil, nil)
-	require.ErrorIs(t, err, ErrNoValueForAutoIncrementalColumn)
+	_, _, err = engine.Exec("INSERT INTO table1(id, title) VALUES (1, 'name2')", nil, nil)
+	require.ErrorIs(t, err, store.ErrKeyAlreadyExists)
 
-	_, _, err = engine.Exec("UPSERT INTO table1(id, title) VALUES (2, 'name2')", nil, nil)
-	require.ErrorIs(t, err, store.ErrKeyNotFound)
+	_, _, err = engine.Exec("INSERT INTO table1(id, title) VALUES (1, 'name2') ON CONFLICT DO NOTHING", nil, nil)
+	require.NoError(t, err)
 
 	_, _, err = engine.Exec("UPSERT INTO table1(id, title) VALUES (1, 'name11')", nil, nil)
 	require.NoError(t, err)
 
-	_, ctxs, err = engine.Exec("INSERT INTO table1(title) VALUES ('name2')", nil, nil)
+	_, _, err = engine.Exec("INSERT INTO table1(id, title) VALUES (2, 'name2')", nil, nil)
+	require.NoError(t, err)
+
+	_, _, err = engine.Exec("UPSERT INTO table1(id, title) VALUES (3, 'name3')", nil, nil)
+	require.NoError(t, err)
+
+	_, ctxs, err = engine.Exec("INSERT INTO table1(title) VALUES ('name4')", nil, nil)
+	require.NoError(t, err)
+	require.Len(t, ctxs, 1)
+	require.True(t, ctxs[0].closed)
+	require.Equal(t, int64(4), ctxs[0].LastInsertedPKs()["table1"])
+	require.Equal(t, 1, ctxs[0].UpdatedRows())
+
+	_, ctxs, err = engine.Exec(`
+		BEGIN TRANSACTION;
+			INSERT INTO table1(title) VALUES ('name5');
+			INSERT INTO table1(title) VALUES ('name6');
+		COMMIT;
+	`, nil, nil)
+	require.NoError(t, err)
+	require.Len(t, ctxs, 1)
+	require.True(t, ctxs[0].closed)
+	require.Equal(t, int64(6), ctxs[0].LastInsertedPKs()["table1"])
+	require.Equal(t, 2, ctxs[0].UpdatedRows())
+}
+
+func TestAutoIncrementUniqueIndex(t *testing.T) {
+	st, err := store.Open("sqldata_s_auto_inc", store.DefaultOptions())
+	require.NoError(t, err)
+	defer os.RemoveAll("sqldata_s_auto_inc")
+
+	engine, err := NewEngine(st, DefaultOptions().WithPrefix(sqlPrefix))
+	require.NoError(t, err)
+
+	_, _, err = engine.Exec("CREATE DATABASE db1", nil, nil)
+	require.NoError(t, err)
+
+	err = engine.SetDefaultDatabase("db1")
+	require.NoError(t, err)
+
+	t.Run("invalid use of auto-increment", func(t *testing.T) {
+		_, _, err := engine.Exec("CREATE TABLE table1 (uid VARCHAR[256], number VARCHAR AUTO_INCREMENT, name VARCHAR, PRIMARY KEY uid)", nil, nil)
+		require.ErrorIs(t, err, ErrAutoIncrementWrongType)
+
+		_, _, err = engine.Exec("CREATE TABLE table1 (id INTEGER AUTO_INCREMENT, number INTEGER AUTO_INCREMENT, name VARCHAR, PRIMARY KEY id)", nil, nil)
+		require.ErrorIs(t, err, ErrAutoIncrementMultiple)
+	})
+
+	_, _, err = engine.Exec("CREATE TABLE table1 (uid VARCHAR[256], number INTEGER AUTO_INCREMENT, name VARCHAR, PRIMARY KEY uid)", nil, nil)
+	require.NoError(t, err)
+
+	_, ctxs, err := engine.Exec("INSERT INTO table1(uid, name) VALUES ('AE1R', 'Jhon')", nil, nil)
+	require.NoError(t, err)
+	require.Len(t, ctxs, 1)
+	require.True(t, ctxs[0].closed)
+	require.Equal(t, int64(1), ctxs[0].LastInsertedPKs()["table1"])
+	require.Equal(t, 1, ctxs[0].UpdatedRows())
+
+	_, ctxs, err = engine.Exec("UPSERT INTO table1(uid, name) VALUES ('AE1R', 'Ben')", nil, nil)
+	require.NoError(t, err)
+
+	_, ctxs, err = engine.Exec("INSERT INTO table1(uid, name) VALUES ('BEKZ', 'Marc')", nil, nil)
 	require.NoError(t, err)
 	require.Len(t, ctxs, 1)
 	require.True(t, ctxs[0].closed)
 	require.Equal(t, int64(2), ctxs[0].LastInsertedPKs()["table1"])
 	require.Equal(t, 1, ctxs[0].UpdatedRows())
 
+
 	_, ctxs, err = engine.Exec(`
 		BEGIN TRANSACTION;
-			INSERT INTO table1(title) VALUES ('name3');
-			INSERT INTO table1(title) VALUES ('name4');
+			INSERT INTO table1(uid, name) VALUES ('VBKZ', 'name3');
+			INSERT INTO table1(uid, name) VALUES ('FZKB', 'name4');
 		COMMIT;
 	`, nil, nil)
 	require.NoError(t, err)

embedded/sql/parser.go

@@ -46,6 +46,9 @@ var reservedWords = map[string]int{
 	"ADD":            ADD,
 	"COLUMN":         COLUMN,
 	"INSERT":         INSERT,
+	"CONFLICT":       CONFLICT,
+	"DO":             DO,
+	"NOTHING":        NOTHING,
 	"UPSERT":         UPSERT,
 	"INTO":           INTO,
 	"VALUES":         VALUES,

embedded/sql/sql_grammar.y

@@ -61,11 +61,12 @@ func setResult(l yyLexer, stmts []SQLStmt) {
     pparam int
     update *colUpdate
     updates []*colUpdate
+    onConflict *OnConflictDo
 }
 
 %token CREATE USE DATABASE SNAPSHOT SINCE UP TO TABLE UNIQUE INDEX ON ALTER ADD COLUMN PRIMARY KEY
 %token BEGIN TRANSACTION COMMIT ROLLBACK
-%token INSERT UPSERT INTO VALUES DELETE UPDATE SET
+%token INSERT UPSERT INTO VALUES DELETE UPDATE SET CONFLICT DO NOTHING
 %token SELECT DISTINCT FROM BEFORE TX JOIN HAVING WHERE GROUP BY LIMIT ORDER ASC DESC AS
 %token NOT LIKE IF EXISTS IN IS
 %token AUTO_INCREMENT NULL NPARAM CAST
@@ -125,6 +126,7 @@ func setResult(l yyLexer, stmts []SQLStmt) {
 %type <boolean> opt_if_not_exists opt_auto_increment opt_not_null opt_not
 %type <update> update
 %type <updates> updates
+%type <onConflict> opt_on_conflict
 
 %start sql
 
@@ -234,9 +236,9 @@ one_or_more_ids:
     }
 
 dmlstmt:
-    INSERT INTO tableRef '(' opt_ids ')' VALUES rows
+    INSERT INTO tableRef '(' opt_ids ')' VALUES rows opt_on_conflict
     {
-        $$ = &UpsertIntoStmt{isInsert: true, tableRef: $3, cols: $5, rows: $8}
+        $$ = &UpsertIntoStmt{isInsert: true, tableRef: $3, cols: $5, rows: $8, onConflict: $9}
     }
 |
     UPSERT INTO tableRef '(' ids ')' VALUES rows
@@ -254,6 +256,16 @@ dmlstmt:
         $$ = &UpdateStmt{tableRef: $2, updates: $4, where: $5, indexOn: $6, limit: int($7)}
     }
 
+opt_on_conflict:
+    {
+        $$ = nil
+    }
+|
+    ON CONFLICT DO NOTHING
+    {
+        $$ = &OnConflictDo{}
+    }
+
 updates:
     update
     {
@@ -399,9 +411,9 @@ colsSpec:
     }
 
 colSpec:
-    IDENTIFIER TYPE opt_max_len opt_auto_increment opt_not_null
+    IDENTIFIER TYPE opt_max_len opt_not_null opt_auto_increment
     {
-        $$ = &ColSpec{colName: $1, colType: $2, maxLen: int($3), autoIncrement: $4, notNull: $5}
+        $$ = &ColSpec{colName: $1, colType: $2, maxLen: int($3), notNull: $4, autoIncrement: $5}
     }
 
 opt_max_len: