feat: elevenlabs plugin

packages/agent/src/defaultCharacter.ts

@@ -6,6 +6,7 @@ export const defaultCharacter: Character = {
   plugins: [
     "@elizaos/plugin-anthropic",
     "@elizaos/plugin-openai",
+    "@elizaos/plugin-elevenlabs",
     // "@elizaos/plugin-local-ai",
     "@elizaos/plugin-discord",
     "@elizaos/plugin-node",
@@ -535,4 +536,4 @@ export const defaultCharacter: Character = {
     "provocative",
   ],
   extends: [],
-};
+};

packages/plugin-elevenlabs/.gitignore

@@ -0,0 +1 @@
+dist/

packages/plugin-elevenlabs/.npmignore

@@ -0,0 +1,6 @@
+*
+
+!dist/**
+!package.json
+!readme.md
+!tsup.config.ts

packages/plugin-elevenlabs/biome.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.5.3/schema.json",
+  "extends": ["../../biome.json"],
+  "organizeImports": {
+    "enabled": false
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space",
+    "indentWidth": 4,
+    "lineWidth": 100
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "single"
+    }
+  },
+  "files": {
+    "ignore": [
+      "dist/**/*",
+      "extra/**/*",
+      "node_modules/**/*"
+    ]
+  }
+}

packages/plugin-elevenlabs/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@elizaos/plugin-elevenlabs",
+  "version": "1.0.0-alpha.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@elizaos/core": "workspace:*",
+    "tsup": "8.3.5"
+  },
+  "scripts": {
+    "build": "tsup --format esm --dts",
+    "dev": "tsup --format esm --dts --watch",
+    "test": "vitest run"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "agentConfig": {
+    "pluginType": "elizaos:plugin:1.0.0",
+    "pluginParameters": {
+      "ELEVENLABS_XI_API_KEY": {
+        "type": "string",
+        "description": "API key for ElevenLabs."
+      },
+      "ELEVENLABS_VOICE_ID": {
+        "type": "string",
+        "description": "Optional. Voice selection ID."
+      },
+      "ELEVENLABS_MODEL_ID": {
+        "type": "string",
+        "description": "Optional. Speech model ID."
+      },
+      "ELEVENLABS_VOICE_STABILITY": {
+        "type": "string",
+        "description": "Optional. Controls voice stability."
+      },
+      "ELEVENLABS_OPTIMIZE_STREAMING_LATENCY": {
+        "type": "string",
+        "description": "Optional. Adjusts streaming latency."
+      },
+      "ELEVENLABS_OUTPUT_FORMAT": {
+        "type": "string",
+        "description": "Optional. Output format (e.g., pcm_16000)."
+      },
+      "ELEVENLABS_VOICE_SIMILARITY_BOOST": {
+        "type": "string",
+        "description": "Optional. Adjusts similarity to the reference voice (0-1)."
+      },
+      "ELEVENLABS_VOICE_STYLE": {
+        "type": "string",
+        "description": "Optional. Controls voice style intensity (0-1)."
+      },
+      "ELEVENLABS_VOICE_USE_SPEAKER_BOOST": {
+        "type": "string",
+        "description": "Optional. Enhances speaker presence (true/false)."
+      }
+    }
+  }
+}

packages/plugin-elevenlabs/src/index.ts

@@ -0,0 +1,100 @@
+import { IAgentRuntime, Plugin, logger, ModelClass } from "@elizaos/core";
+import { Readable } from "node:stream";
+import { prependWavHeader } from "./utils.ts";
+
+function getVoiceSettings(runtime: IAgentRuntime) {
+  const getSetting = (key: string, fallback = "") =>
+    process.env[key] || runtime.getSetting(key) || fallback;
+
+  return {
+    apiKey: getSetting("ELEVENLABS_XI_API_KEY"),
+    voiceId: getSetting("ELEVENLABS_VOICE_ID"),
+    model: getSetting("ELEVENLABS_MODEL_ID", "eleven_monolingual_v1"),
+    stability: getSetting("ELEVENLABS_VOICE_STABILITY", "0.5"),
+    latency: getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY", "0"),
+    outputFormat: getSetting("ELEVENLABS_OUTPUT_FORMAT", "pcm_16000"),
+    similarity: getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST", "0.75"),
+    style: getSetting("ELEVENLABS_VOICE_STYLE", "0"),
+    speakerBoost: getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST", "true"),
+  };
+}
+
+async function fetchSpeech(runtime: IAgentRuntime, text: string) {
+  const settings = getVoiceSettings(runtime);
+  try {
+    const response = await fetch(
+      `https://api.elevenlabs.io/v1/text-to-speech/${settings.voiceId}/stream?optimize_streaming_latency=${settings.latency}&output_format=${settings.outputFormat}`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "xi-api-key": settings.apiKey,
+        },
+        body: JSON.stringify({
+          model_id: settings.model,
+          text,
+          voice_settings: {
+            similarity_boost: settings.similarity,
+            stability: settings.stability,
+            style: settings.style,
+            use_speaker_boost: settings.speakerBoost,
+          },
+        }),
+      }
+    );
+    if (response.status !== 200) {
+      const errorBodyString = await response.text();
+      const errorBody = JSON.parse(errorBodyString);
+
+      if (response.status === 401 && errorBody.detail?.status === "quota_exceeded") {
+        logger.log("ElevenLabs quota exceeded");
+        throw new Error("QUOTA_EXCEEDED");
+      }
+      throw new Error(`Received status ${response.status} from Eleven Labs API: ${JSON.stringify(errorBody)}`);
+    }
+    return Readable.fromWeb(response.body);
+  } catch (error) {
+    logger.error(error);
+    return new Readable({ read() {} });
+  }
+}
+
+export const elevenLabsPlugin: Plugin = {
+  name: "elevenLabs",
+  description: "ElevenLabs plugin",
+  models: {
+    [ModelClass.TEXT_TO_SPEECH]: async (runtime, text) => {
+      const stream = await fetchSpeech(runtime, text);
+      return getVoiceSettings(runtime).outputFormat.startsWith("pcm_")
+        ? prependWavHeader(stream, 1024 * 1024 * 100, parseInt(getVoiceSettings(runtime).outputFormat.slice(4)), 1, 16)
+        : stream;
+    },
+  },
+  tests: [
+    {
+      name: "test eleven labs",
+      tests: [
+        {
+          name: "Eleven Labs API key validation",
+          fn: async (runtime: IAgentRuntime) => {
+            if (!getVoiceSettings(runtime).apiKey) {
+              throw new Error("Missing API key: Please provide a valid Eleven Labs API key.");
+            }
+          },
+        },
+        {
+          name: "Eleven Labs API response",
+          fn: async (runtime: IAgentRuntime) => {
+            try {
+              await fetchSpeech(runtime, "test");
+            } catch(error) {
+              throw new Error(`Failed to fetch speech from Eleven Labs API: ${error.message || "Unknown error occurred"}`);
+            }
+
+          },
+        },
+      ],
+    },
+  ],
+};
+export default elevenLabsPlugin;

packages/plugin-elevenlabs/src/utils.ts

@@ -0,0 +1,53 @@
+import { PassThrough } from "stream";
+import { Readable } from "node:stream";
+
+export function getWavHeader(
+  audioLength: number,
+  sampleRate: number,
+  channelCount = 1,
+  bitsPerSample = 16
+): Buffer {
+  const wavHeader = Buffer.alloc(44);
+  wavHeader.write("RIFF", 0);
+  wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8
+  wavHeader.write("WAVE", 8);
+  wavHeader.write("fmt ", 12);
+  wavHeader.writeUInt32LE(16, 16); // Length of format data
+  wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM)
+  wavHeader.writeUInt16LE(channelCount, 22); // Number of channels
+  wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate
+  wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate
+  wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8)
+  wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample
+  wavHeader.write("data", 36); // Data chunk header
+  wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size
+  return wavHeader;
+}
+
+export function prependWavHeader(
+  readable: Readable,
+  audioLength: number,
+  sampleRate: number,
+  channelCount = 1,
+  bitsPerSample = 16
+): Readable {
+  const wavHeader = getWavHeader(
+    audioLength,
+    sampleRate,
+    channelCount,
+    bitsPerSample
+  );
+  let pushedHeader = false;
+  const passThrough = new PassThrough();
+  readable.on("data", (data) => {
+    if (!pushedHeader) {
+      passThrough.push(wavHeader);
+      pushedHeader = true;
+    }
+    passThrough.push(data);
+  });
+  readable.on("end", () => {
+    passThrough.end();
+  });
+  return passThrough;
+}

packages/plugin-elevenlabs/tsconfig.json

@@ -0,0 +1,25 @@
+{
+    "compilerOptions": {
+        "outDir": "dist",
+        "rootDir": "src",
+        "lib": ["ESNext"],
+        "target": "ESNext",
+        "module": "Preserve",
+        "moduleResolution": "Bundler",
+        "strict": false,
+        "esModuleInterop": true,
+        "skipLibCheck": true,
+        "forceConsistentCasingInFileNames": false,
+        "allowImportingTsExtensions": true,
+        "declaration": true,
+        "emitDeclarationOnly": true,
+        "resolveJsonModule": true,
+        "noImplicitAny": false,
+        "allowJs": true,
+        "checkJs": false,
+        "noEmitOnError": false,
+        "moduleDetection": "force",
+        "allowArbitraryExtensions": true
+    },
+    "include": ["src/**/*.ts"]
+}

packages/plugin-elevenlabs/tsup.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+    entry: ["src/index.ts"],
+    outDir: "dist",
+    sourcemap: true,
+    clean: true,
+    format: ["esm"],
+    external: [
+    ],
+});

packages/plugin-elevenlabs/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+    test: {
+        globals: true,
+        environment: 'node',
+    },
+});