From d5af5a442f44fafbd604266728e4ebe6bdbecc62 Mon Sep 17 00:00:00 2001 From: Ting Chien Meng Date: Wed, 12 Feb 2025 13:48:40 +0800 Subject: [PATCH 1/5] elevenlabs plugin --- packages/plugin-elevenlabs/.gitignore | 1 + packages/plugin-elevenlabs/.npmignore | 6 + packages/plugin-elevenlabs/README.md | 0 packages/plugin-elevenlabs/biome.json | 25 +++ packages/plugin-elevenlabs/package.json | 73 ++++++++ packages/plugin-elevenlabs/src/index.ts | 197 ++++++++++++++++++++ packages/plugin-elevenlabs/tsconfig.json | 25 +++ packages/plugin-elevenlabs/tsup.config.ts | 11 ++ packages/plugin-elevenlabs/vitest.config.ts | 8 + 9 files changed, 346 insertions(+) create mode 100644 packages/plugin-elevenlabs/.gitignore create mode 100644 packages/plugin-elevenlabs/.npmignore create mode 100644 packages/plugin-elevenlabs/README.md create mode 100644 packages/plugin-elevenlabs/biome.json create mode 100644 packages/plugin-elevenlabs/package.json create mode 100644 packages/plugin-elevenlabs/src/index.ts create mode 100644 packages/plugin-elevenlabs/tsconfig.json create mode 100644 packages/plugin-elevenlabs/tsup.config.ts create mode 100644 packages/plugin-elevenlabs/vitest.config.ts diff --git a/packages/plugin-elevenlabs/.gitignore b/packages/plugin-elevenlabs/.gitignore new file mode 100644 index 00000000000..77738287f0e --- /dev/null +++ b/packages/plugin-elevenlabs/.gitignore @@ -0,0 +1 @@ +dist/ \ No newline at end of file diff --git a/packages/plugin-elevenlabs/.npmignore b/packages/plugin-elevenlabs/.npmignore new file mode 100644 index 00000000000..078562eceab --- /dev/null +++ b/packages/plugin-elevenlabs/.npmignore @@ -0,0 +1,6 @@ +* + +!dist/** +!package.json +!readme.md +!tsup.config.ts \ No newline at end of file diff --git a/packages/plugin-elevenlabs/README.md b/packages/plugin-elevenlabs/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/plugin-elevenlabs/biome.json b/packages/plugin-elevenlabs/biome.json new file mode 100644 index 00000000000..d7e0349301a --- /dev/null +++ b/packages/plugin-elevenlabs/biome.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.5.3/schema.json", + "extends": ["../../biome.json"], + "organizeImports": { + "enabled": false + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 4, + "lineWidth": 100 + }, + "javascript": { + "formatter": { + "quoteStyle": "single" + } + }, + "files": { + "ignore": [ + "dist/**/*", + "extra/**/*", + "node_modules/**/*" + ] + } +} \ No newline at end of file diff --git a/packages/plugin-elevenlabs/package.json b/packages/plugin-elevenlabs/package.json new file mode 100644 index 00000000000..a3d83e3adda --- /dev/null +++ b/packages/plugin-elevenlabs/package.json @@ -0,0 +1,73 @@ +{ + "name": "@elizaos/plugin-elevenlabs", + "version": "1.0.0-alpha.0", + "type": "module", + "main": "dist/index.js", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } + }, + "files": [ + "dist" + ], + "dependencies": { + "@elizaos/core": "workspace:*", + "tsup": "8.3.5" + }, + "scripts": { + "build": "tsup --format esm --dts", + "dev": "tsup --format esm --dts --watch", + "test": "vitest run" + }, + "publishConfig": { + "access": "public" + }, + "agentConfig": { + "pluginType": "elizaos:plugin:1.0.0", + "pluginParameters": { + "ELEVENLABS_XI_API_KEY": { + "type": "string", + "description": "API key for ElevenLabs." + }, + "ELEVENLABS_VOICE_ID": { + "type": "string", + "description": "Optional. Voice selection ID." + }, + "ELEVENLABS_MODEL_ID": { + "type": "string", + "description": "Optional. Speech model ID." + }, + "ELEVENLABS_VOICE_STABILITY": { + "type": "string", + "description": "Optional. Controls voice stability." + }, + "ELEVENLABS_OPTIMIZE_STREAMING_LATENCY": { + "type": "string", + "description": "Optional. Adjusts streaming latency." + }, + "ELEVENLABS_OUTPUT_FORMAT": { + "type": "string", + "description": "Optional. Output format (e.g., pcm_16000)." + }, + "ELEVENLABS_VOICE_SIMILARITY_BOOST": { + "type": "string", + "description": "Optional. Adjusts similarity to the reference voice (0-1)." + }, + "ELEVENLABS_VOICE_STYLE": { + "type": "string", + "description": "Optional. Controls voice style intensity (0-1)." + }, + "ELEVENLABS_VOICE_USE_SPEAKER_BOOST": { + "type": "string", + "description": "Optional. Enhances speaker presence (true/false)." + } + } + } +} diff --git a/packages/plugin-elevenlabs/src/index.ts b/packages/plugin-elevenlabs/src/index.ts new file mode 100644 index 00000000000..71f66fef6ce --- /dev/null +++ b/packages/plugin-elevenlabs/src/index.ts @@ -0,0 +1,197 @@ +import { IAgentRuntime, Plugin, logger } from "@elizaos/core"; +import { Readable } from "node:stream"; +import { ModelClass } from "@elizaos/core"; +import { PassThrough } from "stream"; + +export function getWavHeader( + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Buffer { + const wavHeader = Buffer.alloc(44); + wavHeader.write("RIFF", 0); + wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 + wavHeader.write("WAVE", 8); + wavHeader.write("fmt ", 12); + wavHeader.writeUInt32LE(16, 16); // Length of format data + wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) + wavHeader.writeUInt16LE(channelCount, 22); // Number of channels + wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate + wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate + wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) + wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample + wavHeader.write("data", 36); // Data chunk header + wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size + return wavHeader; +} + +function prependWavHeader( + readable: Readable, + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Readable { + const wavHeader = getWavHeader( + audioLength, + sampleRate, + channelCount, + bitsPerSample + ); + let pushedHeader = false; + const passThrough = new PassThrough(); + readable.on("data", (data) => { + if (!pushedHeader) { + passThrough.push(wavHeader); + pushedHeader = true; + } + passThrough.push(data); + }); + readable.on("end", () => { + passThrough.end(); + }); + return passThrough; +} + +function getVoiceSettings(runtime: IAgentRuntime) { + return { + elevenlabsApiKey: + process.env.ELEVENLABS_XI_API_KEY || + runtime.getSetting("ELEVENLABS_XI_API_KEY"), + elevenlabsVoiceId: + process.env.ELEVENLABS_VOICE_ID || + runtime.getSetting("ELEVENLABS_VOICE_ID"), + elevenlabsModel: + process.env.ELEVENLABS_MODEL_ID || + runtime.getSetting("ELEVENLABS_MODEL_ID") || + "eleven_monolingual_v1", + elevenlabsStability: + process.env.ELEVENLABS_VOICE_STABILITY || + runtime.getSetting("ELEVENLABS_VOICE_STABILITY") || + "0.5", + elevenStreamingLatency: + process.env.ELEVENLABS_OPTIMIZE_STREAMING_LATENCY || + runtime.getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY") || + "0", + elevenlabsOutputFormat: + process.env.ELEVENLABS_OUTPUT_FORMAT || + runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT") || + "pcm_16000", + elevenlabsVoiceSimilarity: + process.env.ELEVENLABS_VOICE_SIMILARITY_BOOST || + runtime.getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST") || + "0.75", + elevenlabsVoiceStyle: + process.env.ELEVENLABS_VOICE_STYLE || + runtime.getSetting("ELEVENLABS_VOICE_STYLE") || + "0", + elevenlabsVoiceUseSpeakerBoost: + process.env.ELEVENLABS_VOICE_USE_SPEAKER_BOOST || + runtime.getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST") || + "true", + }; +} + +export const elevenLabsPlugin: Plugin = { + name: "elevenLabs", + description: "ElevenLabs plugin", + + models: { + [ModelClass.TEXT_TO_SPEECH]: async ( + runtime: IAgentRuntime, + text: string | null + ) => { + const { + elevenlabsApiKey, + elevenlabsVoiceId, + elevenlabsModel, + elevenlabsStability, + elevenStreamingLatency, + elevenlabsOutputFormat, + elevenlabsVoiceSimilarity, + elevenlabsVoiceStyle, + elevenlabsVoiceUseSpeakerBoost, + } = getVoiceSettings(runtime); + + try { + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${elevenlabsVoiceId}/stream?optimize_streaming_latency=${elevenStreamingLatency}&output_format=${elevenlabsOutputFormat}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "xi-api-key": elevenlabsApiKey, + }, + body: JSON.stringify({ + model_id: elevenlabsModel, + text: text, + voice_settings: { + similarity_boost: elevenlabsVoiceSimilarity, + stability: elevenlabsStability, + style: elevenlabsVoiceStyle, + use_speaker_boost: elevenlabsVoiceUseSpeakerBoost, + }, + }), + } + ); + + const status = response.status; + if (status !== 200) { + const errorBodyString = await response.text(); + const errorBody = JSON.parse(errorBodyString); + + // Check for quota exceeded error + if (status === 401 && errorBody.detail?.status === "quota_exceeded") { + logger.log("ElevenLabs quota exceeded"); + throw new Error("QUOTA_EXCEEDED"); + } + + throw new Error( + `Received status ${status} from Eleven Labs API: ${errorBodyString}` + ); + } + + if (response) { + const webStream = ReadableStream.from( + response.body as ReadableStream + ); + const reader = webStream.getReader(); + + const readable = new Readable({ + read() { + reader.read().then(({ done, value }) => { + if (done) { + this.push(null); + } else { + this.push(value); + } + }); + }, + }); + + if (elevenlabsOutputFormat.startsWith("pcm_")) { + const sampleRate = Number.parseInt( + elevenlabsOutputFormat.substring(4) + ); + const withHeader = prependWavHeader( + readable, + 1024 * 1024 * 100, + sampleRate, + 1, + 16 + ); + return withHeader; + } else { + return readable; + } + } else { + return new Readable({ + read() {}, + }); + } + } catch (error) {} + }, + }, +}; +export default elevenLabsPlugin; diff --git a/packages/plugin-elevenlabs/tsconfig.json b/packages/plugin-elevenlabs/tsconfig.json new file mode 100644 index 00000000000..2153cf41345 --- /dev/null +++ b/packages/plugin-elevenlabs/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleResolution": "Bundler", + "strict": false, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": false, + "allowImportingTsExtensions": true, + "declaration": true, + "emitDeclarationOnly": true, + "resolveJsonModule": true, + "noImplicitAny": false, + "allowJs": true, + "checkJs": false, + "noEmitOnError": false, + "moduleDetection": "force", + "allowArbitraryExtensions": true + }, + "include": ["src/**/*.ts"] +} \ No newline at end of file diff --git a/packages/plugin-elevenlabs/tsup.config.ts b/packages/plugin-elevenlabs/tsup.config.ts new file mode 100644 index 00000000000..76f9797e298 --- /dev/null +++ b/packages/plugin-elevenlabs/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + outDir: "dist", + sourcemap: true, + clean: true, + format: ["esm"], + external: [ + ], +}); diff --git a/packages/plugin-elevenlabs/vitest.config.ts b/packages/plugin-elevenlabs/vitest.config.ts new file mode 100644 index 00000000000..adbf7255380 --- /dev/null +++ b/packages/plugin-elevenlabs/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + }, +}); From 6a9c76a1fbbaa4f48070b58ad797eb48c7e10751 Mon Sep 17 00:00:00 2001 From: Ting Chien Meng Date: Wed, 12 Feb 2025 13:54:10 +0800 Subject: [PATCH 2/5] clean code --- packages/plugin-elevenlabs/src/index.ts | 53 +------------------------ packages/plugin-elevenlabs/src/utils.ts | 53 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 52 deletions(-) create mode 100644 packages/plugin-elevenlabs/src/utils.ts diff --git a/packages/plugin-elevenlabs/src/index.ts b/packages/plugin-elevenlabs/src/index.ts index 71f66fef6ce..91dc27f3130 100644 --- a/packages/plugin-elevenlabs/src/index.ts +++ b/packages/plugin-elevenlabs/src/index.ts @@ -1,58 +1,7 @@ import { IAgentRuntime, Plugin, logger } from "@elizaos/core"; import { Readable } from "node:stream"; import { ModelClass } from "@elizaos/core"; -import { PassThrough } from "stream"; - -export function getWavHeader( - audioLength: number, - sampleRate: number, - channelCount = 1, - bitsPerSample = 16 -): Buffer { - const wavHeader = Buffer.alloc(44); - wavHeader.write("RIFF", 0); - wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 - wavHeader.write("WAVE", 8); - wavHeader.write("fmt ", 12); - wavHeader.writeUInt32LE(16, 16); // Length of format data - wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) - wavHeader.writeUInt16LE(channelCount, 22); // Number of channels - wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate - wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate - wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) - wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample - wavHeader.write("data", 36); // Data chunk header - wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size - return wavHeader; -} - -function prependWavHeader( - readable: Readable, - audioLength: number, - sampleRate: number, - channelCount = 1, - bitsPerSample = 16 -): Readable { - const wavHeader = getWavHeader( - audioLength, - sampleRate, - channelCount, - bitsPerSample - ); - let pushedHeader = false; - const passThrough = new PassThrough(); - readable.on("data", (data) => { - if (!pushedHeader) { - passThrough.push(wavHeader); - pushedHeader = true; - } - passThrough.push(data); - }); - readable.on("end", () => { - passThrough.end(); - }); - return passThrough; -} +import { prependWavHeader } from "./utils.ts"; function getVoiceSettings(runtime: IAgentRuntime) { return { diff --git a/packages/plugin-elevenlabs/src/utils.ts b/packages/plugin-elevenlabs/src/utils.ts new file mode 100644 index 00000000000..85fa8b18218 --- /dev/null +++ b/packages/plugin-elevenlabs/src/utils.ts @@ -0,0 +1,53 @@ +import { PassThrough } from "stream"; +import { Readable } from "node:stream"; + +export function getWavHeader( + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Buffer { + const wavHeader = Buffer.alloc(44); + wavHeader.write("RIFF", 0); + wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 + wavHeader.write("WAVE", 8); + wavHeader.write("fmt ", 12); + wavHeader.writeUInt32LE(16, 16); // Length of format data + wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) + wavHeader.writeUInt16LE(channelCount, 22); // Number of channels + wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate + wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate + wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) + wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample + wavHeader.write("data", 36); // Data chunk header + wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size + return wavHeader; +} + +export function prependWavHeader( + readable: Readable, + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Readable { + const wavHeader = getWavHeader( + audioLength, + sampleRate, + channelCount, + bitsPerSample + ); + let pushedHeader = false; + const passThrough = new PassThrough(); + readable.on("data", (data) => { + if (!pushedHeader) { + passThrough.push(wavHeader); + pushedHeader = true; + } + passThrough.push(data); + }); + readable.on("end", () => { + passThrough.end(); + }); + return passThrough; +} From e0641c02a7e9db51d99b2b6064bd24a5b4458c5c Mon Sep 17 00:00:00 2001 From: Ting Chien Meng Date: Wed, 12 Feb 2025 16:22:11 +0800 Subject: [PATCH 3/5] add evenlabs --- packages/agent/src/defaultCharacter.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/defaultCharacter.ts b/packages/agent/src/defaultCharacter.ts index 2f3a570de06..0c845eb77ad 100644 --- a/packages/agent/src/defaultCharacter.ts +++ b/packages/agent/src/defaultCharacter.ts @@ -6,9 +6,11 @@ export const defaultCharacter: Character = { plugins: [ "@elizaos/plugin-anthropic", "@elizaos/plugin-openai", + "@elizaos/plugin-elevenlabs", "@elizaos/plugin-local-ai", "@elizaos/plugin-discord", "@elizaos/plugin-node", + "@elizaos/plugin-elevenlabs", "@elizaos/plugin-telegram", "@elizaos/plugin-twitter", ], @@ -535,4 +537,4 @@ export const defaultCharacter: Character = { "provocative", ], extends: [], -}; \ No newline at end of file +}; From 0059bf9cabb77f6cdd62179b5692f3bde7883d08 Mon Sep 17 00:00:00 2001 From: Ting Chien Meng Date: Wed, 12 Feb 2025 16:22:54 +0800 Subject: [PATCH 4/5] remove duplicate one --- packages/agent/src/defaultCharacter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/agent/src/defaultCharacter.ts b/packages/agent/src/defaultCharacter.ts index 0c845eb77ad..0e568e252f2 100644 --- a/packages/agent/src/defaultCharacter.ts +++ b/packages/agent/src/defaultCharacter.ts @@ -10,7 +10,6 @@ export const defaultCharacter: Character = { "@elizaos/plugin-local-ai", "@elizaos/plugin-discord", "@elizaos/plugin-node", - "@elizaos/plugin-elevenlabs", "@elizaos/plugin-telegram", "@elizaos/plugin-twitter", ], From b1b9fdfbac671ab70b81c7e05e78bf8fc8bf78e9 Mon Sep 17 00:00:00 2001 From: Ting Chien Meng Date: Fri, 14 Feb 2025 01:45:14 +0800 Subject: [PATCH 5/5] add test suite --- packages/plugin-elevenlabs/src/index.ts | 214 ++++++++++-------------- 1 file changed, 84 insertions(+), 130 deletions(-) diff --git a/packages/plugin-elevenlabs/src/index.ts b/packages/plugin-elevenlabs/src/index.ts index 91dc27f3130..b8dbc0e84d8 100644 --- a/packages/plugin-elevenlabs/src/index.ts +++ b/packages/plugin-elevenlabs/src/index.ts @@ -1,146 +1,100 @@ -import { IAgentRuntime, Plugin, logger } from "@elizaos/core"; +import { IAgentRuntime, Plugin, logger, ModelClass } from "@elizaos/core"; import { Readable } from "node:stream"; -import { ModelClass } from "@elizaos/core"; import { prependWavHeader } from "./utils.ts"; function getVoiceSettings(runtime: IAgentRuntime) { + const getSetting = (key: string, fallback = "") => + process.env[key] || runtime.getSetting(key) || fallback; + return { - elevenlabsApiKey: - process.env.ELEVENLABS_XI_API_KEY || - runtime.getSetting("ELEVENLABS_XI_API_KEY"), - elevenlabsVoiceId: - process.env.ELEVENLABS_VOICE_ID || - runtime.getSetting("ELEVENLABS_VOICE_ID"), - elevenlabsModel: - process.env.ELEVENLABS_MODEL_ID || - runtime.getSetting("ELEVENLABS_MODEL_ID") || - "eleven_monolingual_v1", - elevenlabsStability: - process.env.ELEVENLABS_VOICE_STABILITY || - runtime.getSetting("ELEVENLABS_VOICE_STABILITY") || - "0.5", - elevenStreamingLatency: - process.env.ELEVENLABS_OPTIMIZE_STREAMING_LATENCY || - runtime.getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY") || - "0", - elevenlabsOutputFormat: - process.env.ELEVENLABS_OUTPUT_FORMAT || - runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT") || - "pcm_16000", - elevenlabsVoiceSimilarity: - process.env.ELEVENLABS_VOICE_SIMILARITY_BOOST || - runtime.getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST") || - "0.75", - elevenlabsVoiceStyle: - process.env.ELEVENLABS_VOICE_STYLE || - runtime.getSetting("ELEVENLABS_VOICE_STYLE") || - "0", - elevenlabsVoiceUseSpeakerBoost: - process.env.ELEVENLABS_VOICE_USE_SPEAKER_BOOST || - runtime.getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST") || - "true", + apiKey: getSetting("ELEVENLABS_XI_API_KEY"), + voiceId: getSetting("ELEVENLABS_VOICE_ID"), + model: getSetting("ELEVENLABS_MODEL_ID", "eleven_monolingual_v1"), + stability: getSetting("ELEVENLABS_VOICE_STABILITY", "0.5"), + latency: getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY", "0"), + outputFormat: getSetting("ELEVENLABS_OUTPUT_FORMAT", "pcm_16000"), + similarity: getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST", "0.75"), + style: getSetting("ELEVENLABS_VOICE_STYLE", "0"), + speakerBoost: getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST", "true"), }; } +async function fetchSpeech(runtime: IAgentRuntime, text: string) { + const settings = getVoiceSettings(runtime); + try { + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${settings.voiceId}/stream?optimize_streaming_latency=${settings.latency}&output_format=${settings.outputFormat}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "xi-api-key": settings.apiKey, + }, + body: JSON.stringify({ + model_id: settings.model, + text, + voice_settings: { + similarity_boost: settings.similarity, + stability: settings.stability, + style: settings.style, + use_speaker_boost: settings.speakerBoost, + }, + }), + } + ); + if (response.status !== 200) { + const errorBodyString = await response.text(); + const errorBody = JSON.parse(errorBodyString); + + if (response.status === 401 && errorBody.detail?.status === "quota_exceeded") { + logger.log("ElevenLabs quota exceeded"); + throw new Error("QUOTA_EXCEEDED"); + } + throw new Error(`Received status ${response.status} from Eleven Labs API: ${JSON.stringify(errorBody)}`); + } + return Readable.fromWeb(response.body); + } catch (error) { + logger.error(error); + return new Readable({ read() {} }); + } +} + export const elevenLabsPlugin: Plugin = { name: "elevenLabs", description: "ElevenLabs plugin", - models: { - [ModelClass.TEXT_TO_SPEECH]: async ( - runtime: IAgentRuntime, - text: string | null - ) => { - const { - elevenlabsApiKey, - elevenlabsVoiceId, - elevenlabsModel, - elevenlabsStability, - elevenStreamingLatency, - elevenlabsOutputFormat, - elevenlabsVoiceSimilarity, - elevenlabsVoiceStyle, - elevenlabsVoiceUseSpeakerBoost, - } = getVoiceSettings(runtime); - - try { - const response = await fetch( - `https://api.elevenlabs.io/v1/text-to-speech/${elevenlabsVoiceId}/stream?optimize_streaming_latency=${elevenStreamingLatency}&output_format=${elevenlabsOutputFormat}`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - "xi-api-key": elevenlabsApiKey, - }, - body: JSON.stringify({ - model_id: elevenlabsModel, - text: text, - voice_settings: { - similarity_boost: elevenlabsVoiceSimilarity, - stability: elevenlabsStability, - style: elevenlabsVoiceStyle, - use_speaker_boost: elevenlabsVoiceUseSpeakerBoost, - }, - }), - } - ); - - const status = response.status; - if (status !== 200) { - const errorBodyString = await response.text(); - const errorBody = JSON.parse(errorBodyString); - - // Check for quota exceeded error - if (status === 401 && errorBody.detail?.status === "quota_exceeded") { - logger.log("ElevenLabs quota exceeded"); - throw new Error("QUOTA_EXCEEDED"); - } - - throw new Error( - `Received status ${status} from Eleven Labs API: ${errorBodyString}` - ); - } - - if (response) { - const webStream = ReadableStream.from( - response.body as ReadableStream - ); - const reader = webStream.getReader(); - - const readable = new Readable({ - read() { - reader.read().then(({ done, value }) => { - if (done) { - this.push(null); - } else { - this.push(value); - } - }); - }, - }); - - if (elevenlabsOutputFormat.startsWith("pcm_")) { - const sampleRate = Number.parseInt( - elevenlabsOutputFormat.substring(4) - ); - const withHeader = prependWavHeader( - readable, - 1024 * 1024 * 100, - sampleRate, - 1, - 16 - ); - return withHeader; - } else { - return readable; - } - } else { - return new Readable({ - read() {}, - }); - } - } catch (error) {} + [ModelClass.TEXT_TO_SPEECH]: async (runtime, text) => { + const stream = await fetchSpeech(runtime, text); + return getVoiceSettings(runtime).outputFormat.startsWith("pcm_") + ? prependWavHeader(stream, 1024 * 1024 * 100, parseInt(getVoiceSettings(runtime).outputFormat.slice(4)), 1, 16) + : stream; }, }, + tests: [ + { + name: "test eleven labs", + tests: [ + { + name: "Eleven Labs API key validation", + fn: async (runtime: IAgentRuntime) => { + if (!getVoiceSettings(runtime).apiKey) { + throw new Error("Missing API key: Please provide a valid Eleven Labs API key."); + } + }, + }, + { + name: "Eleven Labs API response", + fn: async (runtime: IAgentRuntime) => { + try { + await fetchSpeech(runtime, "test"); + } catch(error) { + throw new Error(`Failed to fetch speech from Eleven Labs API: ${error.message || "Unknown error occurred"}`); + } + + }, + }, + ], + }, + ], }; export default elevenLabsPlugin;