diff --git a/core/build/esbuildMod.ts b/core/build/esbuildMod.ts index 29dbae31d..978b55575 100644 --- a/core/build/esbuildMod.ts +++ b/core/build/esbuildMod.ts @@ -1,6 +1,5 @@ import { FunctionComponent, Page } from "../server/types.ts"; import { build } from "./build.ts"; -import { esbuild } from "./deps.ts"; export class EsbuildMod { #elementName: string; @@ -13,8 +12,4 @@ export class EsbuildMod { build = async () => { return await build(this.#elementName); }; - - stop = () => { - esbuild.stop(); - }; } diff --git a/core/map/mod.test.ts b/core/map/mod.test.ts index 59e558901..7acd1c012 100644 --- a/core/map/mod.test.ts +++ b/core/map/mod.test.ts @@ -74,8 +74,8 @@ Deno.test("Store: keys", () => { assertEquals(entries, ["key6"]); }); -export function processMap(map: Store): Store { - const result = new Store(); +function processMap(map: Store): Map { + const result = new Map(); map.forEach((value, key) => { result.set(key, value * 2); }); @@ -88,7 +88,7 @@ Deno.test("Store: forEach", () => { inputMap.set("two", 2); inputMap.set("three", 3); - const expectedOutput = new Store(); + const expectedOutput = new Map(); expectedOutput.set("one", 2); expectedOutput.set("two", 4); expectedOutput.set("three", 6); @@ -134,14 +134,10 @@ const store = new Store({ branch: "store", token, }); -const i = store.sync(5000); -Deno.test("Store: save it to github", async () => { - store.set("key1", time); - const r = await store.commit(); - assertEquals(r?.data.content?.name, "records.json"); -}); -Deno.test("Store: get value from github", async () => { +Deno.test("Store: set and get value from github", async () => { + store.set("key1", time); + await store.commit(); const g = await store.get("key1"); assertEquals(g, time); }); @@ -157,12 +153,6 @@ Deno.test("Store: sync with github periodically", async () => { assertEquals(g, 2); }); -Deno.test("Store: destroy map", async () => { - await store.destroy(); - const g = await store.get("key1"); - assertEquals(g, undefined); -}); - Deno.test("Store: destroy map without options", async () => { try { const s = new Store(); @@ -172,6 +162,12 @@ Deno.test("Store: destroy map without options", async () => { } }); +Deno.test("Store: destroy map", async () => { + await store.destroy(); + const g = await store.get("key1"); + assertEquals(g, undefined); +}); + const s = new Store({ owner: "fastrodev", repo: "fastro", @@ -190,10 +186,24 @@ Deno.test("Store: sync exist file", async () => { }); await newStore.get("exist"); const intervalId = newStore.sync(); - await new Promise((resolve) => setTimeout(resolve, 15000)); + await new Promise((resolve) => setTimeout(resolve, 10000)); const r = await newStore.get("exist"); assertEquals(r, true); clearInterval(intervalId); }); -if (i) clearInterval(i); +Deno.test("Store: sync, same size after multiple commit", async () => { + const newStore = new Store({ + owner: "fastrodev", + repo: "fastro", + path: "modules/store/map.json", + branch: "store", + token, + }); + await Promise.all([ + newStore.set("user", "zaid").commit(), + newStore.set("city", "pare").commit(), + newStore.set("country", "indonesia").commit(), + ]); + assertEquals(newStore.size(), 4); +}); diff --git a/core/map/mod.ts b/core/map/mod.ts index a2a5e6936..b0eaa76cf 100644 --- a/core/map/mod.ts +++ b/core/map/mod.ts @@ -5,12 +5,13 @@ import { type StoreOptions, uploadFileToGitHub, } from "../../utils/octokit.ts"; +import { createTaskQueue } from "../../utils/queue.ts"; export class Store { private map: Map; private options: StoreOptions; private intervalId: number | null = null; - private isCommitting: boolean = false; + private taskQueue = createTaskQueue(); constructor(options: StoreOptions = null) { this.map = new Map(); @@ -130,73 +131,81 @@ export class Store { }); } - /** - * Save to github - */ - async commit() { - if (this.isCommitting) { - throw new Error("Commit in progress, please wait."); - } - if (!this.options) throw new Error("Options are needed to commit"); - this.isCommitting = true; - this.cleanUpExpiredEntries(); - try { - return await this.saveToGitHub( - { - token: this.options.token, - owner: this.options.owner, - repo: this.options.repo, - path: this.options.path, - branch: this.options.branch, - }, - ); - } finally { - this.isCommitting = false; - } - } - - /** - * Delete file from repository - */ - async destroy() { - if (!this.options) throw new Error("Options are needed to destroy."); - if (this.intervalId) clearInterval(this.intervalId); - this.map.clear(); - return await deleteGithubFile({ + private async joinMaps() { + if (!this.options) return this.map; + const remoteMap = await getMap({ token: this.options.token, owner: this.options.owner, repo: this.options.repo, path: this.options.path, branch: this.options.branch, }); + if (!remoteMap) return this.map; + + // deno-lint-ignore no-explicit-any + for (const [key, entry] of remoteMap.entries() as any) { + if (!this.map.has(key)) this.map.set(key, entry); + } + + this.cleanUpExpiredEntries(); + return this.map; } + /** + * Save to github with queue + */ + commit = async () => { + return await this.taskQueue.process(this.commiting, this.options); + }; + + private commiting = async (options?: StoreOptions) => { + if (!options) return; + await this.joinMaps(); + return await this.saveToGitHub( + { + token: options?.token, + owner: options.owner, + repo: options.repo, + path: options.path, + branch: options?.branch, + }, + ); + }; + /** * Save the map to the repository periodically at intervals * @param interval * @returns intervalId */ - sync(interval: number = 3000) { + sync(interval: number = 5000) { if (this.intervalId) clearInterval(this.intervalId); - this.intervalId = setInterval(async () => { - if (!this.options || (this.map.size === 0)) { - return; + this.intervalId = setInterval(() => { + if (this.map.size === 0) return; + this.taskQueue.process(this.commiting, this.options); + }, interval); + return this.intervalId; + } + + /** + * Delete file from repository + */ + async destroy() { + try { + if (!this.options) { + throw new Error("Options are needed to destroy."); } - this.isCommitting = true; - const r = await this.saveToGitHub({ + if (this.intervalId) clearInterval(this.intervalId); + this.map.clear(); + return await deleteGithubFile({ token: this.options.token, owner: this.options.owner, repo: this.options.repo, path: this.options.path, branch: this.options.branch, }); - console.log(JSON.stringify({ - sha: r.data.content?.sha, - path: r.data.content?.path, - })); - this.isCommitting = false; - }, interval); - return this.intervalId; + } catch (error) { + throw error; + } } private async syncMap() { diff --git a/core/server/deps.ts b/core/server/deps.ts index 7776e9198..6de7e6fe3 100644 --- a/core/server/deps.ts +++ b/core/server/deps.ts @@ -6,14 +6,14 @@ export * from "jsr:@std/path@^1.0.1"; export { encodeHex } from "jsr:@std/encoding@^1.0.5/hex"; export { assert, assertEquals, assertExists } from "jsr:@std/assert@^1.0.6"; -export { h } from "https://esm.sh/preact@10.24.1"; +export { h } from "https://esm.sh/preact@10.24.2"; export type { ComponentChild, ComponentChildren, JSX, VNode, -} from "https://esm.sh/preact@10.24.1"; +} from "https://esm.sh/preact@10.24.2"; export { renderToString, renderToStringAsync, -} from "https://esm.sh/preact-render-to-string@6.5.9?deps=preact@10.24.1"; +} from "https://esm.sh/preact-render-to-string@6.5.9?deps=preact@10.24.2"; diff --git a/deno.json b/deno.json index 0dec0ba7f..73e3d78f6 100644 --- a/deno.json +++ b/deno.json @@ -12,7 +12,7 @@ }, "imports": { "@app/": "./", - "preact": "npm:preact@^10.24.1" + "preact": "npm:preact@^10.24.2" }, "compilerOptions": { "jsx": "react-jsx", diff --git a/modules/blog/blog.json b/modules/blog/blog.json index 0309e1c6f..c8c87fcbd 100644 --- a/modules/blog/blog.json +++ b/modules/blog/blog.json @@ -1,4 +1,9 @@ [ + { + "title": "Using Queues to Avoid Race Conditions", + "url": "/blog/queue", + "date": "10/05/2024" + }, { "title": "Store: Key and Value Map with TTL", "url": "/blog/store", diff --git a/modules/index/index.launchpad.tsx b/modules/index/index.launchpad.tsx index 4820edc82..adf9e7fc6 100644 --- a/modules/index/index.launchpad.tsx +++ b/modules/index/index.launchpad.tsx @@ -1,371 +1,398 @@ -function StackSvg() { +export default function Launchpad() { return ( - - - - - - - ); -} +
+ + -function LogisticSvg() { - return ( - - - - - - - - + +
); } -function NoteSvg() { - return ( - - - - - - - - ); -} +// function StackSvg() { +// return ( +// +// +// +// +// +// +// ); +// } -function FormSvg() { - return ( - - - - - - - - - - ); -} +// function LogisticSvg() { +// return ( +// +// +// +// +// +// +// +// +// ); +// } -function HtmlSvg() { - return ( - - - - - - - - - - - ); -} +// function NoteSvg() { +// return ( +// +// +// +// +// +// +// +// ); +// } -function SurveySvg() { - return ( - - - - - - ); -} +// function FormSvg() { +// return ( +// +// +// +// +// +// +// +// +// +// ); +// } -function BlogSvg() { - return ( - - - - - - - - - - - - ); -} +// function HtmlSvg() { +// return ( +// +// +// +// +// +// +// +// +// +// +// ); +// } -function SocialSvg() { - return ( - - - - - - - - - - - ); -} +// function SurveySvg() { +// return ( +// +// +// +// +// +// ); +// } -function StoreSvg() { - return ( - - - - - - - - - ); -} +// function BlogSvg() { +// return ( +// +// +// +// +// +// +// +// +// +// +// +// ); +// } -function LoyalSvg() { - return ( - - - - - - ); -} +// function SocialSvg() { +// return ( +// +// +// +// +// +// +// +// +// +// +// ); +// } -function AttendanceSvg() { - return ( - - - - - - - - - - - ); -} +// function StoreSvg() { +// return ( +// +// +// +// +// +// +// +// +// ); +// } -function PurchaseSvg() { - return ( - - - - - - - - - ); -} +// function LoyalSvg() { +// return ( +// +// +// +// +// +// ); +// } -import Search from "@app/components/search.tsx"; -import ProjectBox from "@app/components/project-box.tsx"; -import AdminSvg from "@app/components/icons/admin-svg.tsx"; -import AdsSvg from "@app/components/icons/ads-svg.tsx"; -import SalesSvg from "@app/components/icons/sales-svg.tsx"; -import WareHouseSvg from "@app/components/icons/warehouse-svg.tsx"; +// function AttendanceSvg() { +// return ( +// +// +// +// +// +// +// +// +// +// +// ); +// } -export default function Launchpad() { - return ( -
- -
- - -

Admin

-
- - -

Advertising

-
- - -

Attendance

-
- - -

Blog

-
- - -

Landing Page

-
- - -

Logistic

-
- - -

Loyalty

-
- - -

Medical Record

-
- - -

Purchasing

-
- - -

Registration

-
- - -

Sales

-
- - -

Social Media

-
- - -

Store

-
- - -

Survey

-
- - -

Warehouse

-
- - -

Visitor queue

-
-
-
- ); -} +// function PurchaseSvg() { +// return ( +// +// +// +// +// +// +// +// +// ); +// } + +// import Search from "@app/components/search.tsx"; +// import ProjectBox from "@app/components/project-box.tsx"; +// import AdminSvg from "@app/components/icons/admin-svg.tsx"; +// import AdsSvg from "@app/components/icons/ads-svg.tsx"; +// import SalesSvg from "@app/components/icons/sales-svg.tsx"; +// import WareHouseSvg from "@app/components/icons/warehouse-svg.tsx"; + +// function Board() { +// return ( +//
+// +//
+// +// +//

Admin

+//
+// +// +//

Advertising

+//
+// +// +//

Attendance

+//
+// +// +//

Blog

+//
+// +// +//

Landing Page

+//
+// +// +//

Logistic

+//
+// +// +//

Loyalty

+//
+// +// +//

Medical Record

+//
+// +// +//

Purchasing

+//
+// +// +//

Registration

+//
+// +// +//

Sales

+//
+// +// +//

Social Media

+//
+// +// +//

Store

+//
+// +// +//

Survey

+//
+// +// +//

Warehouse

+//
+// +// +//

Visitor queue

+//
+//
+//
+// ); +// } diff --git a/modules/index/mod.ts b/modules/index/mod.ts index 49cbd39aa..eb0b317ae 100644 --- a/modules/index/mod.ts +++ b/modules/index/mod.ts @@ -47,8 +47,8 @@ export default function (s: Fastro) { baseUrl: Deno.env.get("ENV") === "DEVELOPMENT" ? "http://localhost:8000" : "https://fastro.dev", - new: "Store: Key and Value Map with TTL", - destination: "blog/store", + new: "Using Queues to Avoid Race Conditions", + destination: "blog/queue", isLogin: ses?.isLogin, avatar_url: ses?.avatar_url, html_url: ses?.html_url, diff --git a/modules/message/mod.test.ts b/modules/message/mod.test.ts new file mode 100644 index 000000000..ae9ff047e --- /dev/null +++ b/modules/message/mod.test.ts @@ -0,0 +1,15 @@ +import { assertEquals } from "@app/core/server/deps.ts"; +import { createMessage } from "@app/modules/message/mod.ts"; +import { ulid } from "jsr:@std/ulid"; + +Deno.test("Message: create message", async () => { + const userId = ulid(); + + const res = await createMessage({ + userId, + content: "Hello", + title: "hi", + }); + + assertEquals(res?.userId, userId); +}); diff --git a/modules/message/mod.ts b/modules/message/mod.ts new file mode 100644 index 000000000..abad9c30e --- /dev/null +++ b/modules/message/mod.ts @@ -0,0 +1,30 @@ +import { uploadFileToGitHub } from "@app/utils/octokit.ts"; +import { Store } from "@app/core/map/mod.ts"; +import { ulid } from "jsr:@std/ulid/ulid"; + +export async function createMessage( + options: { userId: string; content: string; title: string }, +) { + const store = new Store({ + token: Deno.env.get("GITHUB_TOKEN"), + owner: Deno.env.get("MESSAGE_OWNER") || "fastrodev", + repo: Deno.env.get("MESSAGE_REPO") || "fastro", + branch: Deno.env.get("MESSAGE_BRANCH") || "store", + path: `modules/store/${options.userId}/message.json`, + }); + + const postId = ulid(); + await store.set(postId, { title: options.title, postId }).commit(); + + const r = await uploadFileToGitHub({ + token: Deno.env.get("GITHUB_TOKEN"), + owner: Deno.env.get("MESSAGE_OWNER") || "fastrodev", + repo: Deno.env.get("MESSAGE_REPO") || "fastro", + branch: Deno.env.get("MESSAGE_BRANCH") || "store", + path: `modules/store/${options.userId}/${postId}.md`, + content: options.content, + }); + + if (r.status !== 201) return; + return { postId, ...options }; +} diff --git a/post/queue.md b/post/queue.md new file mode 100644 index 000000000..511af961a --- /dev/null +++ b/post/queue.md @@ -0,0 +1,69 @@ +--- +title: "Using Queues to Avoid Race Conditions" +description: "This article explores how using queues can prevent race conditions in concurrent programming by ensuring orderly access to shared resources, enhancing data integrity and application reliability." +author: Admin +date: 10/05/2024 +--- + +This article explores how using queues can prevent race conditions in concurrent +programming by ensuring orderly access to shared resources, enhancing data +integrity and application reliability. + +```ts +import { assertEquals } from "@app/core/server/deps.ts"; +import { createTaskQueue } from "@app/utils/queue.ts"; + +Deno.test("Queue: create message", async () => { + const q = createTaskQueue(); + const x = await q.process(() => "x"); + const y = await q.process(() => 1); + const z = await q.process(() => true); + const o = await q.process(() => ({})); + const v = await q.process(() => {}); + assertEquals(x, "x"); + assertEquals(y, 1); + assertEquals(z, true); + assertEquals(o, {}); + assertEquals(v, undefined); +}); +``` + +We have already implemented queue mechanisms in our store to effectively manage +concurrent access to shared resources, allowing you to commit multiple data +entries in parallel. + +```ts +Deno.test("Store: sync, same size after multiple commit", async () => { + const newStore = new Store({ + owner: "fastrodev", + repo: "fastro", + path: "modules/store/map.json", + branch: "store", + token, + }); + await Promise.all([ + newStore.set("user", "zaid").commit(), + newStore.set("gender", "male").commit(), + newStore.set("city", "pare").commit(), + newStore.set("country", "indonesia").commit(), + ]); + assertEquals(newStore.size(), 4); +}); +``` + +This approach has allowed us to: + +- Enhance Performance: By organizing requests in a queue, we can process them + sequentially, reducing the likelihood of conflicts and improving overall + system performance. +- Maintain Data Integrity: With controlled access to shared resources, we + minimize the risk of data corruption, ensuring that transactions are completed + accurately. +- Improve User Experience: Customers experience smoother interactions as their + requests are handled in an orderly fashion, leading to faster response times + and increased satisfaction. +- Scalability: Our queue system allows us to scale operations efficiently, + accommodating higher volumes of transactions without compromising performance. + +By leveraging this strategy, we have successfully mitigated race conditions, +resulting in a more reliable and efficient store environment. diff --git a/utils/queue.test.ts b/utils/queue.test.ts new file mode 100644 index 000000000..fd5297ff6 --- /dev/null +++ b/utils/queue.test.ts @@ -0,0 +1,16 @@ +import { assertEquals } from "@app/core/server/deps.ts"; +import { createTaskQueue } from "@app/utils/queue.ts"; + +Deno.test("Queue: create message", async () => { + const q = createTaskQueue(); + const x = await q.process(() => "x"); + const y = await q.process(() => 1); + const z = await q.process(() => true); + const o = await q.process(() => ({})); + const v = await q.process(() => {}); + assertEquals(x, "x"); + assertEquals(y, 1); + assertEquals(z, true); + assertEquals(o, {}); + assertEquals(v, undefined); +}); diff --git a/utils/queue.ts b/utils/queue.ts new file mode 100644 index 000000000..fd1560afc --- /dev/null +++ b/utils/queue.ts @@ -0,0 +1,34 @@ +export function createTaskQueue() { + // deno-lint-ignore no-explicit-any + const array: any = []; + + let isExecuting = false; + async function exec() { + while (array.length > 0) { + const { fn, args, resolve } = array.shift(); + try { + const result = await fn(...args); + resolve(result); + } catch (error) { + console.error("Error executing function:", error); + } + } + isExecuting = false; + } + + // deno-lint-ignore no-explicit-any + function process( + fn: (...args: T) => Promise | R, + ...args: T + ): Promise { + return new Promise((resolve) => { + array.push({ fn, args, resolve }); + if (!isExecuting) { + isExecuting = true; + exec(); + } + }); + } + + return { process: process }; +}