diff --git a/README.md b/README.md index 877655d..0cc12fe 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Support for `Date`, `RegExp` and `symbol` as part of keys. Support for `symbol` A powerful `db.find` function that supports approximate matching and works on both indexes and regular keys with over 50 operators including regular expressions, soundex/echoes, credit card, SSNs and more. If something is missing, it can be added in as little as one line. +A subscription API allows listening for changes to the database. + # Usage ```javascript @@ -35,8 +37,6 @@ Primitive keys are automatically converted to the arrays required by Deno KV. Deno KV does not provide a `db.clear` function. Denodata does. -```javascript - ```javascript await db.set("mykey", "myvalue"); await (async () => { const {key,value,version} = await db.get("mykey")})(); @@ -129,6 +129,12 @@ await (async () => { console.log(value); // prints Book instance, less efficient, large index scans because of lack of cname } })(); +db.subscribe({set: {key:1,value:1}},async (value) => { + console.log(value); // prints {key:1,value:1} so long as db.set(1,1) is called +}); +db.subscribe({delete: (key) => key<10 ? key : undefined},async (value) => { + console.log(value); // prints key so long as db.delete(key) is called with value less than 10 +}); ``` # Installation @@ -242,6 +248,11 @@ Notes: - Works like `Deno KV.set`. Does not manage indexes or do specialized serialization. - See notes at start of API section regarding `key` and `value` types. +`db.subscribe(on:{delete?:any,patch?:any,put?:any,set?:any},callback:(arg1:any,arg2?:any)=>void,{cname?:string,metadata?:{[key:string]:any}}):void` + +- Calls the `callback` with the same signature as `db.delete`, `db.patch`, `db.put`, or `db.set` if the value or key and value in the `on` pattern, as well as `cname` and `metadata` match the call to `db.delete`, `db.patch`, `db.put`, or `db.set`. + + # Key and Value Space ## Keys @@ -431,7 +442,7 @@ The following operators are supported in patterns. # Testing - `constants.js ... 100.000% (3/3)` -- `denodata.ts ... index.js ... 90.196% (690/765)` +- `denodata.ts ... 82.809% (684/826)` - `operators.ts ... ... 95.330% (347/364)` # Release History (Reverse Chronological Order) @@ -440,6 +451,10 @@ The following operators are supported in patterns. - Beta commenced when unit test coverage first exceeded 90% - The exposed API is stable. Additional features may be exposed. + +2023-08-09 v0.0.27 (Beta) + - Exposed subscription API. Test coverage dropped below 90% as a result. + 2023-07-28 v0.0.26 (Beta) - Documentation corrections. diff --git a/package.json b/package.json index 21a1550..7e6dd03 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "denodata", - "version": "0.0.26", - "description": "Generalized indexing and search for Deno KV", + "version": "0.0.27", + "description": "Generalized indexing, search, and change subscriptions for Deno KV", "type": "module", "main": "mod.ts", "scripts": { diff --git a/src/denodata.test.js b/src/denodata.test.js index 9c99ca3..93e29a9 100644 --- a/src/denodata.test.js +++ b/src/denodata.test.js @@ -460,6 +460,70 @@ const test = async (deno) => { expect(sum.value.value).toEqual(1n); }) + deno.test("subscribe delete", async (t) => { + return new Promise(async (resolve,reject) => { + await db.subscribe({delete: 1},(value) => { + try { + expect(value).toEqual(1); + resolve(); + } catch(e) { + reject(e); + } + }); + await db.delete(1); + }); + }) + + deno.test("subscribe delete - function test", async (t) => { + return new Promise(async (resolve,reject) => { + await db.subscribe({delete: (value) => value===1 ? value : undefined},(value) => { + try { + expect(value).toEqual(1); + resolve(); + } catch(e) { + reject(e); + } + }); + await db.delete(1); + }); + }) + + deno.test("subscribe set", async (t) => { + return new Promise(async (resolve,reject) => { + db.subscribe({set: {key:1,value:1}},async (key,value) => { + try { + expect(key).toEqual(1); + expect(value).toEqual(1); + resolve(); + } catch(e) { + reject(e); + } finally { + await db.delete(1); + } + }); + await db.set(1,1); + }); + }) + + deno.test("subscribe set - function test", async (t) => { + return new Promise(async (resolve,reject) => { + db.subscribe({set: {key:1,value:(value) => value===1 ? value : undefined}},async (key,value) => { + try { + expect(key).toEqual(1); + expect(value).toEqual(1); + resolve(); + } catch(e) { + reject(e); + } finally { + await db.delete(1); + } + }); + await db.set(1,1); + }); + }) + + + return async () => { const errors = [], start = Date.now(); diff --git a/src/denodata.ts b/src/denodata.ts index 7b27324..f0b3b02 100644 --- a/src/denodata.ts +++ b/src/denodata.ts @@ -57,7 +57,7 @@ const isId = (value:any) => { } type ScoredFunction = Function & {score?:number}; -const matchKeys:ScoredFunction = (pattern:Array, target:Array) => { +const matchKeys:ScoredFunction = (pattern:Array, target:Array):boolean => { matchKeys.score = 1; return pattern.every((item, i) => { const type = typeof (item); @@ -89,6 +89,12 @@ const matchKeys:ScoredFunction = (pattern:Array, target:Array) => { const selector = (value:any,pattern:any,{root=value,parent,key}:{root?:any,parent?:object,key?:string}={}) : any => { const ptype = typeof(pattern), vtype = typeof(value); + if(value===pattern) { + return value; + } + if(ptype==="number" && vtype==="number") { + return isNaN(pattern) && isNaN(value) ? value : undefined + } if(ptype==="function") { return pattern(value,{root,parent,key}); } @@ -137,8 +143,6 @@ const selector = (value:any,pattern:any,{root=value,parent,key}:{root?:any,paren } return value; } - if(ptype==="number" && vtype==="number" && isNaN(pattern) && isNaN(value)) return value; - return pattern===value ? value : undefined; } const serializeKey = (key:any,skip:any[]=["bigint"]) => { @@ -328,7 +332,11 @@ Denodata.prototype.clear = async function () { const _delete = db.delete.bind(db); Denodata.prototype.delete = async function (value:any, {cname, indexOnly,find}:{cname?:string,indexOnly?:boolean,find?:boolean} = {}) { - const type = typeof (value); + const type = typeof (value), + originalValue:any = value; + if(type!=="object" && !Array.isArray(value)) { + value = [value] + } if (Array.isArray( value)) { const key = toPattern(toKey(value)); if (value.length === 1 && isId(value[0])) { @@ -371,50 +379,49 @@ Denodata.prototype.delete = async function (value:any, {cname, indexOnly,find}:{ } else if (value && type === "object") { if(value instanceof Uint8Array) { await _delete([value]); - return - } - if(value instanceof RegExp || value instanceof Date) { + } else if(value instanceof RegExp || value instanceof Date) { await _delete(toKey(value)); - return - } - const id = value[this.options.idProperty]; - if (id) { - cname ||= getCname(id) || value.constructor.name; - value = serializeValue(value); - const indexes:({indexType:string,indexKeys:string[]})[] = []; - Object.values(this.schema[(cname as string)]?.indexes || {}).forEach((value) => { - const {indexType, keys} = (value as {[key:string]:any}); - indexes.push({indexType, indexKeys:keys}) - }); - for (const {indexType, indexKeys} of indexes) { - const keys = serializeKey(this.getKeys(value, indexKeys, {indexType, cname})), - indexPrefix = toIndexPrefix(indexType); - let keyBatch = keys.splice(0, this.options.maxTransactionSize); - while (keyBatch.length > 0) { - const tn = this.atomic(); - for (let key of keyBatch) { - key = this.options.indexValueMutator(key,cname); - if(key && !key.some((item:any) => item==null)) { - tn.delete([indexPrefix, ...key, id]); + } else { + const id = value[this.options.idProperty]; + if (id) { + cname ||= getCname(id) || value.constructor.name; + const metadata = value[this.options.metadataProperty]; + value = serializeValue(value); + const indexes: ({ indexType: string, indexKeys: string[] })[] = []; + Object.values(this.schema[(cname as string)]?.indexes || {}).forEach((value) => { + const {indexType, keys} = (value as { [key: string]: any }); + indexes.push({indexType, indexKeys: keys}) + }); + for (const {indexType, indexKeys} of indexes) { + const keys = serializeKey(this.getKeys(value, indexKeys, {indexType, cname})), + indexPrefix = toIndexPrefix(indexType); + let keyBatch = keys.splice(0, this.options.maxTransactionSize); + while (keyBatch.length > 0) { + const tn = this.atomic(); + for (let key of keyBatch) { + key = this.options.indexValueMutator(key, cname); + if (key && !key.some((item: any) => item == null)) { + tn.delete([indexPrefix, ...key, id]); + } } + await tn.commit(); + keyBatch = keys.splice(0, this.options.maxTransactionSize); } - await tn.commit(); - keyBatch = keys.splice(0, this.options.maxTransactionSize); } + if (!indexOnly) { + await _delete([id]); + } + } else if (find) { + for await (const entry of this.find(value, {cname})) { + await _delete(entry.key); + } + } else { + throw new Error("Can't delete object that does not have an id unless find option is true"); } - if (!indexOnly) { - await _delete([id]); - } - } else if(find) { - for await (const entry of this.find(value,{cname})) { - await _delete(entry.key); - } - } else { - throw new Error("Can't delete object that does not have an id unless find option is true"); } - } else { - await db.delete([value]); } + const args = [deserializeSpecial(null,originalValue), {cname, indexOnly,find}]; + await handleEvent("delete", Object.assign({value:originalValue}, cname ? {cname} : undefined),args); } //valueMatch,select,{cname,fulltext,scan,sort,sortable,minScore,limit=Infinity,offset=0}={} @@ -668,10 +675,12 @@ Denodata.prototype.matchValue = function(pattern:object, target:object) { } Denodata.prototype.patch = async function (value:any, {cname,pattern,metadata}:{cname?:string|undefined,pattern?:object|undefined,metadata?:object|undefined} = {}) { - const type = typeof (value); + const type = typeof (value), + originalValue = value; if (value && type==="object" && !(value[this.options.idProperty] || pattern)) { throw new TypeError(`Can't patch non-object or object without id key if there is no pattern.`); } + cname ||= getCname(value[this.options.idProperty]); if(value && type==="object") { try { value = structuredClone(value); @@ -696,7 +705,6 @@ Denodata.prototype.patch = async function (value:any, {cname,pattern,metadata}:{ } if(metadata) value[this.options.metadataProperty] = metadata; value = serializeValue(value); - cname ||= getCname(value[this.options.idProperty]); const indexes: any[] = []; Object.values(this.schema[(cname as string)]?.indexes || {}).forEach((value) => { const {type,keys} = (value as {[key:string]:any}); @@ -705,44 +713,48 @@ Denodata.prototype.patch = async function (value:any, {cname,pattern,metadata}:{ const id = value[this.options.idProperty], entry = await this.get([id]), patched = entry.value || {}; + let result; if (indexes.length === 0) { Object.assign(patched, value) - await this.put(patched); - return id; - } - for (const {indexType, indexKeys} of indexes) { - const oldIndexKeys = this.getKeys(patched, indexKeys, {indexType, cname}), - newIndexKeys = this.getKeys(value, indexKeys, {indexType, cname}), - removeKeys = oldIndexKeys.reduce((removeKeys:any[], oldKey:any[]) => { - if (newIndexKeys.some((newKey:any[]) => matchKeys(oldKey, newKey))) { + result = this.put(patched); + } else { + for (const {indexType, indexKeys} of indexes) { + const oldIndexKeys = this.getKeys(patched, indexKeys, {indexType, cname}), + newIndexKeys = this.getKeys(value, indexKeys, {indexType, cname}), + removeKeys = oldIndexKeys.reduce((removeKeys: any[], oldKey: any[]) => { + if (newIndexKeys.some((newKey: any[]) => matchKeys(oldKey, newKey))) { + return removeKeys; + } + removeKeys.push(serializeKey(oldKey)); return removeKeys; + }, []), + indexPrefix = toIndexPrefix(indexType); + let keys = removeKeys.splice(0, this.options.maxTransactionSize); + while (keys.length > 0) { + const tn = this.atomic(); + for (let key of keys) { + key = this.options.indexValueMutator(key, cname); + if (key && !key.some((item: any) => item == null)) { + tn.delete([indexPrefix, ...key, id]); + } } - removeKeys.push(serializeKey(oldKey)); - return removeKeys; - }, []), - indexPrefix = toIndexPrefix(indexType); - let keys = removeKeys.splice(0, this.options.maxTransactionSize); - while (keys.length > 0) { - const tn = this.atomic(); - for (let key of keys) { - key = this.options.indexValueMutator(key,cname); - if(key && !key.some((item:any)=>item==null)) { - tn.delete([indexPrefix, ...key, id]); - } + await tn.commit(); + keys = removeKeys.splice(0, this.options.maxTransactionSize); } - await tn.commit(); - keys = removeKeys.splice(0, this.options.maxTransactionSize); } + if (metadata) { // prepare to patch metadata separately + delete value[this.options.metadataProperty]; + } + Object.assign(patched, value); + if (metadata) { // patch metadata separately + patched[this.options.metadataProperty] ||= {}; + Object.assign(patched[this.options.metadataProperty], metadata); + } + result = this.put(patched, {cname, patch: true}); // should put inside a transaction } - if(metadata) { // prepare to patch metadata separately - delete value[this.options.metadataProperty]; - } - Object.assign(patched, value); - if(metadata) { // patch metadata separately - patched[this.options.metadataProperty] ||= {}; - Object.assign(patched[this.options.metadataProperty], metadata); - } - return await this.put(patched, {cname,patch:true}); // should put inside a transaction + const args = [deserializeSpecial(null,originalValue), {cname,pattern,metadata}]; + await handleEvent("patch",Object.assign( {value},cname ? {cname} : undefined,metadata ? {metadata} : undefined),args); + return result; } Denodata.prototype.put = async function (object: { [key:string]:any }, {cname, metadata,indexType, autoIndex,indexKeys,patch} :{cname?:string,metadata?:object,indexType?:string,autoIndex?:boolean,indexKeys?:string[],patch?:boolean}={}) : Promise { @@ -764,38 +776,43 @@ Denodata.prototype.put = async function (object: { [key:string]:any }, {cname, m } if (indexes.length === 0) { await this.set([id], object); - return id; - } - if(!patch) { - await this.delete([id], {indexOnly: true}); - } // clears all index entries - for (let i = 0; i < indexes.length; i++) { - let {indexType = "object", indexKeys} = indexes[i]; - const keys = serializeKey(this.getKeys(object, indexKeys ? [indexKeys] : null, {indexType, cname})), - indexPrefix = toIndexPrefix(indexType); - //if(keys.some((key) => key.some((item)=>item==null))) { - // throw new TypeError(`Can't index null or undefined value for keys ${JSON.stringify(indexKeys)} for index ${name} on object ${JSON.stringify(object)}`); - //} - let keyBatch = keys.splice(0, this.options.maxTransactionSize); - while (keyBatch.length > 0) { - const tn = this.atomic(); - for (let key of keyBatch) { - key = this.options.indexValueMutator(key,cname); - if(key && !key.some((item:any)=>item==null)) { - tn.set([indexPrefix, ...key, id], 0); // 0 is correct, index entries are just keys + } else { + if (!patch) { + await this.delete([id], {indexOnly: true}); + } // clears all index entries + for (let i = 0; i < indexes.length; i++) { + let {indexType = "object", indexKeys} = indexes[i]; + const keys = serializeKey(this.getKeys(object, indexKeys ? [indexKeys] : null, {indexType, cname})), + indexPrefix = toIndexPrefix(indexType); + //if(keys.some((key) => key.some((item)=>item==null))) { + // throw new TypeError(`Can't index null or undefined value for keys ${JSON.stringify(indexKeys)} for index ${name} on object ${JSON.stringify(object)}`); + //} + let keyBatch = keys.splice(0, this.options.maxTransactionSize); + while (keyBatch.length > 0) { + const tn = this.atomic(); + for (let key of keyBatch) { + key = this.options.indexValueMutator(key, cname); + if (key && !key.some((item: any) => item == null)) { + tn.set([indexPrefix, ...key, id], 0); // 0 is correct, index entries are just keys + } } + await tn.commit(); + keyBatch = keys.splice(0, this.options.maxTransactionSize); } - await tn.commit(); - keyBatch = keys.splice(0, this.options.maxTransactionSize); } + await this.set([id], object); } - await this.set([id], object); + const args = [object, {cname, metadata,indexType, autoIndex,indexKeys,patch}]; + await handleEvent("put",Object.assign( {value:object},cname ? {cname} : undefined,metadata ? {metadata} : undefined),args); return id; } + const _set = db.set.bind(db); Denodata.prototype.set = async function (key:any, value:any,metadata:object|undefined) { + const originalValue = value; value = {data:value,metadata:metadata||value[this.options.metadataProperty]}; + metadata = value.metadata; //delete value.data[this.options.metadataProperty]; const type = typeof(value.metadata?.expires); if(type==="number") { @@ -805,8 +822,64 @@ Denodata.prototype.set = async function (key:any, value:any,metadata:object|unde } value = serializeValue(value); delete value.data[this.options.metadataProperty]; - return _set(toKey(key),value); + const result = await _set(toKey(key),value), + args = [key, deserializeSpecial(null,originalValue), metadata]; + await handleEvent("set",Object.assign({value:{key,value:deserializeSpecial(null,value.data)}},metadata ? {metadata} : undefined),args); + return result; } +const SUBSCRIPTIONS:{ + [key:string]:Map<{[key:string]: any },(value:any)=>any> +} = { + delete:new Map(), + patch:new Map(), + put:new Map(), + set:new Map() + }; + +async function handleEvent(event:string, value:{[key:string]:any}={},args:any[]=[]) { + for(const entry of SUBSCRIPTIONS[event]) { + const pattern:{[key:string]:any} = entry[0].pattern, + ptype:string = typeof(pattern), + target:{[key:string]:any} = {...value}; + target.pattern = value.value; + delete target.value; + if(ptype==="function" && (pattern as Function)(pattern)) { + await entry[1].apply(null,args as [value:any]); + } else if(selector(target,entry[0])!==undefined) { + await entry[1].apply(null,args as [value:any]); + } + }; +} + +Denodata.prototype.subscribe = function(on:{delete?:any,patch?:any,put?:any,set?:any}={},callback:(value:any)=>void, {cname,metadata}:{cname?:string,metadata?:{[key:string]:any}}={}) { + for(const [key,pattern] of Object.entries(on)) { + SUBSCRIPTIONS[key].set(Object.assign({pattern},cname ? {cname} : undefined,metadata ? {metadata} : undefined),callback); + } +} + +Denodata.prototype.unsubscribe = function(on:{delete?:any,patch?:any,put?:any,set?:any}={},callback?:(value:any)=>void,options:{cname?:string,metadata?:{[key:string]:any}}={}) { + for(const [event,pattern] of Object.entries(on)) { + for (const entry of SUBSCRIPTIONS[event]) { + if (pattern) { + const ptype = typeof (pattern); + if (ptype === "function" && pattern !== entry[0].pattern) { + return; + } + if (pattern !== undefined) { + if (selector(entry[0], {pattern, ...options}) === undefined) { + return; + } + } + } + if (callback && callback !== entry[1]) { + continue; + } + this[event].subscriptions.delete(entry[0]); + } + } +} + + export default Denodata; \ No newline at end of file