Merge branch 'v2-develop' into v2-add-character-methods

elizaOS · Feb 10, 2025 · 2adae43 · 2adae43
2 parents ae6c4c5 + 20a7b47
commit 2adae43
Show file tree

Hide file tree

Showing 72 changed files with 2,708 additions and 4,276 deletions.
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -11,7 +11,7 @@
     "build:cli": "turbo run build --filter=./packages/cli && cd packages/cli && bun link",
     "start": "turbo run start --filter=!./packages/docs",
     "agent": "turbo run start --filter=@elizaos/agent",
-    "dev": "turbo run dev --filter=!./packages/docs",
+    "dev": "turbo run dev --filter=!./packages/docs --concurrency=20",
     "release": "bun run build && bun format && npx lerna publish --no-private --force-publish",
     "docker:build": "bash ./scripts/docker.sh build",
     "docker:run": "bash ./scripts/docker.sh run",

diff --git a/packages/agent/src/api.ts b/packages/agent/src/api.ts
@@ -1,11 +1,11 @@
 import {
-    type AgentRuntime,
     type Character,
-    logger,
     getEnvVariable,
+    IAgentRuntime,
+    logger,
     type UUID,
     validateCharacterConfig,
-    validateUuid,
+    validateUuid
 } from "@elizaos/core";
 import bodyParser from "body-parser";
 import cors from "cors";
@@ -46,7 +46,7 @@ function validateUUIDParams(
 }
 
 export function createApiRouter(
-    agents: Map<string, AgentRuntime>,
+    agents: Map<string, IAgentRuntime>,
     directClient: CharacterServer
 ): express.Router {
     const router = express.Router();
@@ -117,7 +117,7 @@ export function createApiRouter(
         };
         if (!agentId) return;
 
-        const agent: AgentRuntime = agents.get(agentId);
+        const agent: IAgentRuntime = agents.get(agentId);
 
         if (agent) {
             agent.stop();
@@ -134,7 +134,7 @@ export function createApiRouter(
         };
         if (!agentId) return;
 
-        let agent: AgentRuntime = agents.get(agentId);
+        let agent: IAgentRuntime = agents.get(agentId);
 
         // update character
         if (agent) {
@@ -338,7 +338,7 @@ export function createApiRouter(
     router.post("/agents/:agentId/stop", async (req, res) => {
         const agentId = req.params.agentId;
         console.log("agentId", agentId);
-        const agent: AgentRuntime = agents.get(agentId);
+        const agent: IAgentRuntime = agents.get(agentId);
 
         // update character
         if (agent) {

diff --git a/packages/agent/src/defaultCharacter.ts b/packages/agent/src/defaultCharacter.ts
@@ -4,11 +4,11 @@ export const defaultCharacter: Character = {
   name: "Eliza",
   username: "eliza",
   plugins: [
-    "@elizaos/plugin-node",
-    "@elizaos/plugin-bootstrap",
     "@elizaos/plugin-anthropic",
     "@elizaos/plugin-openai",
-    "@elizaos/plugin-local-ai",
+    "@elizaos/plugin-discord",
+    "@elizaos/plugin-node",
+    "elizaos/plugin-telegram",
   ],
   settings: {
     secrets: {},

diff --git a/packages/agent/src/server.ts b/packages/agent/src/server.ts
@@ -1,8 +1,6 @@
 import {
     composeContext,
     logger,
-    generateCaption,
-    generateImage,
     generateMessageResponse,
     generateObject,
     messageCompletionFooter,
@@ -166,10 +164,7 @@ export class CharacterServer {
                     return;
                 }
 
-                const transcription = await runtime.call(ModelClass.TRANSCRIPTION, {
-                    file: fs.createReadStream(audioFile.path),
-                    model: "whisper-1",
-                });
+                const transcription = await runtime.useModel(ModelClass.TRANSCRIPTION, fs.createReadStream(audioFile.path));
 
                 res.json(transcription);
             }
@@ -591,15 +586,11 @@ export class CharacterServer {
                     res.status(404).send("Agent not found");
                     return;
                 }
-
-                const images = await generateImage({ ...req.body }, agent);
+                const images = await agent.useModel(ModelClass.IMAGE, { ...req.body });
                 const imagesRes: { image: string; caption: string }[] = [];
                 if (images.data && images.data.length > 0) {
                     for (let i = 0; i < images.data.length; i++) {
-                        const caption = await generateCaption(
-                            { imageUrl: images.data[i] },
-                            agent
-                        );
+                        const caption = await agent.useModel(ModelClass.IMAGE_DESCRIPTION, images.data[i]);
                         imagesRes.push({
                             image: images.data[i],
                             caption: caption.title,
@@ -823,10 +814,7 @@ export class CharacterServer {
                 // Get the text to convert to speech
                 const textToSpeak = response.text;
 
-                const speechResponse = await runtime.call(ModelClass.TRANSCRIPTION, {
-                    text: textToSpeak,
-                    runtime,
-                });
+                const speechResponse = await runtime.useModel(ModelClass.TEXT_TO_SPEECH, textToSpeak);
 
                 if (!speechResponse.ok) {
                     throw new Error(

diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts
@@ -39,7 +39,7 @@ export const composeContext = ({
     template: TemplateType;
 }) => {
     const templateStr =
-        typeof template === "function" ? template({ state }) : template;
+        composeRandomUser(typeof template === "function" ? template({ state }) : template, 10);
 
     const templateFunction = handlebars.compile(templateStr);
     return templateFunction(state);

diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts
@@ -91,11 +91,15 @@ export async function trimTokens(
   maxTokens: number,
   runtime: IAgentRuntime
 ) {
-  if (!context) return "";
+  if (!context) throw new Error("Trim tokens received a null context");
+
+  // if context is less than of maxtokens / 5, skip
+  if (context.length < (maxTokens / 5)) return context;
+
   if (maxTokens <= 0) throw new Error("maxTokens must be positive");
 
   try {
-      const tokens = await runtime.call(ModelClass.TEXT_TOKENIZER_ENCODE, context);
+      const tokens = await runtime.useModel(ModelClass.TEXT_TOKENIZER_ENCODE, { context });
 
       // If already within limits, return unchanged
       if (tokens.length <= maxTokens) {
@@ -106,7 +110,7 @@ export async function trimTokens(
       const truncatedTokens = tokens.slice(-maxTokens);
 
       // Decode back to text - js-tiktoken decode() returns a string directly
-      return await runtime.call(ModelClass.TEXT_TOKENIZER_DECODE, truncatedTokens);
+      return await runtime.useModel(ModelClass.TEXT_TOKENIZER_DECODE, { tokens: truncatedTokens });
   } catch (error) {
       logger.error("Error in trimTokens:", error);
       // Return truncated string if tokenization fails
@@ -127,7 +131,7 @@ export async function generateText({
   stopSequences?: string[];
   customSystemPrompt?: string;
 }): Promise<string> {
-  const text = await runtime.call(modelClass, {
+  const text = await runtime.useModel(modelClass, {
     runtime,
     context,
     stopSequences,
@@ -262,7 +266,7 @@ export const generateObject = async ({
     throw new Error(errorMessage);
   }
 
-  const { object } = await runtime.call(modelClass, {
+  const { object } = await runtime.useModel(modelClass, {
     runtime,
     context,
     modelClass,
@@ -320,7 +324,7 @@ export async function generateMessageResponse({
   logger.debug("Context:", context);
 
   return await withRetry(async () => {
-    const text = await runtime.call(modelClass, {
+    const text = await runtime.useModel(modelClass, {
     runtime,
       context,
       stop: stopSequences,
@@ -337,60 +341,4 @@ export async function generateMessageResponse({
 
     return parsedContent;
   });
-}
-
-// ================ IMAGE-RELATED FUNCTIONS ================
-export const generateImage = async (
-  data: {
-    prompt: string;
-    width: number;
-    height: number;
-    count?: number;
-    negativePrompt?: string;
-    numIterations?: number;
-    guidanceScale?: number;
-    seed?: number;
-    modelId?: string;
-    jobId?: string;
-    stylePreset?: string;
-    hideWatermark?: boolean;
-    safeMode?: boolean;
-    cfgScale?: number;
-  },
-  runtime: IAgentRuntime
-): Promise<{
-  success: boolean;
-  data?: string[];
-  error?: any;
-}> => {
-  return await withRetry(
-    async () => {
-      const result = await runtime.call(ModelClass.IMAGE, data);
-      return {
-        success: true,
-        data: result.images,
-        error: undefined,
-      };
-    },
-    {
-      maxRetries: 2,
-      initialDelay: 2000,
-    }
-  );
-};
-
-export const generateCaption = async (
-  data: { imageUrl: string },
-  runtime: IAgentRuntime
-): Promise<{
-  title: string;
-  description: string;
-}> => {
-  const { imageUrl } = data;
-  const resp = await runtime.call(ModelClass.IMAGE_DESCRIPTION, imageUrl);
-
-  return {
-    title: resp.title.trim(),
-    description: resp.description.trim(),
-  };
-};
+}
diff --git a/packages/core/src/knowledge.ts b/packages/core/src/knowledge.ts
@@ -31,7 +31,7 @@ async function get(
         return [];
     }
 
-    const embedding = await runtime.call(ModelClass.TEXT_EMBEDDING, processed);
+    const embedding = await runtime.useModel(ModelClass.TEXT_EMBEDDING, processed);
     const fragments = await runtime.knowledgeManager.searchMemories(
         {
             embedding,
@@ -69,7 +69,7 @@ async function set(
     chunkSize = 512,
     bleed = 20
 ) {
-    const embedding = await runtime.call(ModelClass.TEXT_EMBEDDING, null);
+    const embedding = await runtime.useModel(ModelClass.TEXT_EMBEDDING, null);
     await runtime.documentsManager.createMemory({
         id: item.id,
         agentId: runtime.agentId,
@@ -84,7 +84,7 @@ async function set(
     const fragments = await splitChunks(preprocessed, chunkSize, bleed);
 
     for (const fragment of fragments) {
-        const embedding = await runtime.call(ModelClass.TEXT_EMBEDDING, fragment);
+        const embedding = await runtime.useModel(ModelClass.TEXT_EMBEDDING, fragment);
         await runtime.knowledgeManager.createMemory({
             // We namespace the knowledge base uuid to avoid id
             // collision with the document above.

diff --git a/packages/core/src/memory.ts b/packages/core/src/memory.ts
@@ -66,11 +66,11 @@ export class MemoryManager implements IMemoryManager {
 
         try {
             // Generate embedding from text content
-            memory.embedding = await this.runtime.call(ModelClass.TEXT_EMBEDDING, memoryText);
+            memory.embedding = await this.runtime.useModel(ModelClass.TEXT_EMBEDDING, memoryText);
         } catch (error) {
             logger.error("Failed to generate embedding:", error);
             // Fallback to zero vector if embedding fails
-            memory.embedding = await this.runtime.call(ModelClass.TEXT_EMBEDDING, null);
+            memory.embedding = await this.runtime.useModel(ModelClass.TEXT_EMBEDDING, null);
         }
 
         return memory;
@@ -185,7 +185,7 @@ export class MemoryManager implements IMemoryManager {
         logger.log("Creating Memory", memory.id, memory.content.text);
 
         if(!memory.embedding){
-            const embedding = await this.runtime.call(ModelClass.TEXT_EMBEDDING, null);
+            const embedding = await this.runtime.useModel(ModelClass.TEXT_EMBEDDING, null);
             memory.embedding = embedding;
         }