diff --git a/packages/agent/src/helper.ts b/packages/agent/src/helper.ts new file mode 100644 index 00000000000..e0120f7ce8b --- /dev/null +++ b/packages/agent/src/helper.ts @@ -0,0 +1,89 @@ +import { messageCompletionFooter } from "@elizaos/core"; +import path from "node:path"; +import multer from "multer"; +import fs from "node:fs"; + + + +export const messageHandlerTemplate = + // {{goals}} + // "# Action Examples" is already included + `{{actionExamples}} +(Action examples are for reference only. Do not use the information from them in your response.) + +# Knowledge +{{knowledge}} + +# Task: Generate dialog and actions for the character {{agentName}}. +About {{agentName}}: +{{bio}} +{{lore}} + +{{providers}} + +{{attachments}} + +# Capabilities +Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section. + +{{messageDirections}} + +{{recentMessages}} + +{{actions}} + +# Instructions: Write the next message for {{agentName}}. +${messageCompletionFooter}`; + +export const hyperfiHandlerTemplate = `{{actionExamples}} +(Action examples are for reference only. Do not use the information from them in your response.) + +# Knowledge +{{knowledge}} + +# Task: Generate dialog and actions for the character {{agentName}}. +About {{agentName}}: +{{bio}} +{{lore}} + +{{providers}} + +{{attachments}} + +# Capabilities +Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section. + +{{messageDirections}} + +{{recentMessages}} + +{{actions}} + +# Instructions: Write the next message for {{agentName}}. + +Response format should be formatted in a JSON block like this: +\`\`\`json +{ "lookAt": "{{nearby}}" or null, "emote": "{{emotes}}" or null, "say": "string" or null, "actions": (array of strings) or null } +\`\`\` +`; + + + + +export const storage = multer.diskStorage({ + destination: (req, file, cb) => { + const uploadDir = path.join(process.cwd(), "data", "uploads"); + // Create the directory if it doesn't exist + if (!fs.existsSync(uploadDir)) { + fs.mkdirSync(uploadDir, { recursive: true }); + } + cb(null, uploadDir); + }, + filename: (req, file, cb) => { + const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`; + cb(null, `${uniqueSuffix}-${file.originalname}`); + }, +}); + +// some people have more memory than disk.io +export const upload = multer({ storage /*: multer.memoryStorage() */ }); diff --git a/packages/agent/src/server.ts b/packages/agent/src/server.ts index 302b9d9c378..827b9b56f8f 100644 --- a/packages/agent/src/server.ts +++ b/packages/agent/src/server.ts @@ -9,105 +9,31 @@ import { type Content, type Media, type Memory, - type IAgentRuntime + type IAgentRuntime, + type Character } from "@elizaos/core"; import bodyParser from "body-parser"; import cors from "cors"; import express, { type Request as ExpressRequest } from "express"; -import multer from "multer"; + import * as fs from "node:fs"; import * as path from "node:path"; import { z } from "zod"; import { createApiRouter } from "./api.ts"; import replyAction from "./reply.ts"; +import { messageHandlerTemplate } from "./helper.ts"; +import { upload } from "./helper.ts"; -const storage = multer.diskStorage({ - destination: (req, file, cb) => { - const uploadDir = path.join(process.cwd(), "data", "uploads"); - // Create the directory if it doesn't exist - if (!fs.existsSync(uploadDir)) { - fs.mkdirSync(uploadDir, { recursive: true }); - } - cb(null, uploadDir); - }, - filename: (req, file, cb) => { - const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`; - cb(null, `${uniqueSuffix}-${file.originalname}`); - }, -}); - -// some people have more memory than disk.io -const upload = multer({ storage /*: multer.memoryStorage() */ }); - -export const messageHandlerTemplate = - // {{goals}} - // "# Action Examples" is already included - `{{actionExamples}} -(Action examples are for reference only. Do not use the information from them in your response.) - -# Knowledge -{{knowledge}} - -# Task: Generate dialog and actions for the character {{agentName}}. -About {{agentName}}: -{{bio}} -{{lore}} - -{{providers}} - -{{attachments}} - -# Capabilities -Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section. - -{{messageDirections}} - -{{recentMessages}} - -{{actions}} - -# Instructions: Write the next message for {{agentName}}. -${messageCompletionFooter}`; - -export const hyperfiHandlerTemplate = `{{actionExamples}} -(Action examples are for reference only. Do not use the information from them in your response.) - -# Knowledge -{{knowledge}} - -# Task: Generate dialog and actions for the character {{agentName}}. -About {{agentName}}: -{{bio}} -{{lore}} - -{{providers}} - -{{attachments}} - -# Capabilities -Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section. - -{{messageDirections}} - -{{recentMessages}} - -{{actions}} -# Instructions: Write the next message for {{agentName}}. -Response format should be formatted in a JSON block like this: -\`\`\`json -{ "lookAt": "{{nearby}}" or null, "emote": "{{emotes}}" or null, "say": "string" or null, "actions": (array of strings) or null } -\`\`\` -`; export class CharacterServer { public app: express.Application; private agents: Map; // container management private server: any; // Store server instance - public startAgent: Function; // Store startAgent functor - public loadCharacterTryPath: Function; // Store loadCharacterTryPath functor - public jsonToCharacter: Function; // Store jsonToCharacter functor + public startAgent: () => Promise; // Store startAgent function + public loadCharacterTryPath: (characterPath: string) => Promise; // Store loadCharacterTryPath function + public jsonToCharacter: (filePath: string, character: string | never) => Promise; // Store jsonToCharacter function constructor() { logger.log("DirectClient constructor"); @@ -177,7 +103,7 @@ export class CharacterServer { async (req: express.Request, res: express.Response) => { const agentId = req.params.agentId; const roomId = stringToUuid( - req.body.roomId ?? "default-room-" + agentId + req.body.roomId ?? `default-room-${agentId}` ); const userId = stringToUuid(req.body.userId ?? "user"); @@ -248,7 +174,7 @@ export class CharacterServer { }; const memory: Memory = { - id: stringToUuid(messageId + "-" + userId), + id: stringToUuid(`${messageId}-${userId}`), ...userMessage, agentId: runtime.agentId, userId, @@ -494,7 +420,7 @@ export class CharacterServer { if (hfOut.lookAt !== null || hfOut.emote !== null) { contentObj.text += ". Then I "; if (hfOut.lookAt !== null) { - contentObj.text += "looked at " + hfOut.lookAt; + contentObj.text += `looked at ${hfOut.lookAt}`; if (hfOut.emote !== null) { contentObj.text += " and "; } diff --git a/packages/client/src/components/chat.tsx b/packages/client/src/components/chat.tsx index f470063d152..675ff6560fd 100644 --- a/packages/client/src/components/chat.tsx +++ b/packages/client/src/components/chat.tsx @@ -17,7 +17,7 @@ import { useEffect, useRef, useState } from "react"; import AIWriter from "react-aiwriter"; import { AudioRecorder } from "./audio-recorder"; import CopyButton from "./copy-button"; -import { Avatar, AvatarImage } from "./ui/avatar"; +import { Avatar, AvatarFallback, AvatarImage } from "./ui/avatar"; import { Badge } from "./ui/badge"; import ChatTtsButton from "./ui/chat/chat-tts-button"; import { useAutoScroll } from "./ui/chat/hooks/useAutoScroll"; @@ -31,6 +31,70 @@ type ExtraContentFields = { type ContentWithUser = Content & ExtraContentFields; +function MessageContent({ + message, + agentId, +}: { + message: ContentWithUser; + agentId: UUID; +}) { + return ( +
+ + {message.user === "user" ? message.text : {message.text}} + {/* Attachments */} +
+ {message.attachments?.map((attachment: IAttachment) => ( +
+ attachment +
+ + +
+
+ ))} +
+ +
+ {message.text && !message.isLoading ? ( +
+ + +
+ ) : null} +
+ {message.source ? ( + {message.source} + ) : null} + {message.action ? ( + {message.action} + ) : null} + {message.createdAt ? ( + + ) : null} +
+
+
+ ); +} + export default function Page({ agentId }: { agentId: UUID }) { const { toast } = useToast(); const [selectedFile, setSelectedFile] = useState(null); @@ -83,7 +147,7 @@ export default function Page({ agentId }: { agentId: UUID }) { const newMessages = [ { text: input, - user: "{{user1}}", + user: "user", createdAt: Date.now(), attachments, }, @@ -163,9 +227,9 @@ export default function Page({ agentId }: { agentId: UUID }) { disableAutoScroll={disableAutoScroll} > {messages.map((message: ContentWithUser) => { - const variant = getMessageVariant(message?.user); return (
- {message?.user !== "user" ? ( - - - - ) : null} -
- - {message?.user !== "user" ? ( - - {message?.text} - - ) : ( - message?.text - )} - {/* Attachments */} -
- {message?.attachments?.map( - (attachment: IAttachment) => ( -
- attachment -
- - -
-
- ) - )} -
- -
- {message?.text && - !message?.isLoading ? ( -
- - -
- ) : null} -
- {message?.source ? ( - - {message.source} - - ) : null} - {message?.action ? ( - - {message.action} - - ) : null} - {message?.createdAt ? ( - - ) : null} -
-
-
+ {message.user !== "user" ? ( + <> + + + + + + ) : ( + <> + + + + + U + + + + )}
); diff --git a/packages/client/src/components/ui/chat/chat-bubble.tsx b/packages/client/src/components/ui/chat/chat-bubble.tsx index 30c31dbcffa..ac96bc95a19 100644 --- a/packages/client/src/components/ui/chat/chat-bubble.tsx +++ b/packages/client/src/components/ui/chat/chat-bubble.tsx @@ -76,8 +76,8 @@ const chatBubbleMessageVariants = cva("p-4", { variants: { variant: { received: - "bg-secondary text-secondary-foreground rounded-r-lg rounded-tl-lg", - sent: "bg-primary text-primary-foreground rounded-l-lg rounded-tr-lg", + "bg-secondary text-secondary-foreground rounded-lg rounded-bl-none", + sent: "bg-primary text-primary-foreground rounded-lg rounded-br-none" }, layout: { default: "", diff --git a/packages/plugin-local-ai/src/index.ts b/packages/plugin-local-ai/src/index.ts index 8b37c9f818d..a1810b7ad8a 100644 --- a/packages/plugin-local-ai/src/index.ts +++ b/packages/plugin-local-ai/src/index.ts @@ -10,10 +10,10 @@ import { RawImage, type Tensor, } from "@huggingface/transformers"; -import { exec } from "child_process"; +import { exec } from "node:child_process"; import * as Echogarden from "echogarden"; import { EmbeddingModel, FlagEmbedding } from "fastembed"; -import fs from "fs"; +import fs from "node:fs"; import { getLlama, type Llama, @@ -24,11 +24,11 @@ import { type LlamaModel } from "node-llama-cpp"; import { nodewhisper } from "nodejs-whisper"; -import os from "os"; -import path from "path"; -import { PassThrough, Readable } from "stream"; -import { fileURLToPath } from "url"; -import { promisify } from "util"; +import os from "node:os"; +import path from "node:path"; +import { PassThrough, Readable } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; import { z } from "zod"; const execAsync = promisify(exec); @@ -285,9 +285,11 @@ class LocalAIManager { if (!this.sequence) { throw new Error("LLaMA model not initialized"); } - const session = new LlamaChatSession({ contextSequence: this.sequence }); - const wordsToPunishTokens = wordsToPunish.flatMap((word) => this.model!.tokenize(word)); + if (!this.model) { + throw new Error("Model is not initialized"); + } + const wordsToPunishTokens = wordsToPunish.flatMap((word) => this.model.tokenize(word)); const repeatPenalty: LlamaChatSessionRepeatPenalty = { punishTokensFilter: () => wordsToPunishTokens, @@ -447,9 +449,11 @@ export const localAIPlugin: Plugin = { async init(config: Record) { try { const validatedConfig = await configSchema.parseAsync(config); - Object.entries(validatedConfig).forEach(([key, value]) => { - if (value) process.env[key] = value; - }); + for (const [key, value] of Object.entries(validatedConfig)) { + if (value) { + process.env[key] = value; + } + } await localAIManager.initialize(); } catch (error) {