diff --git a/packages/agent/src/defaultCharacter.ts b/packages/agent/src/defaultCharacter.ts index c515c5095f..15e26752ad 100644 --- a/packages/agent/src/defaultCharacter.ts +++ b/packages/agent/src/defaultCharacter.ts @@ -6,6 +6,7 @@ export const defaultCharacter: Character = { plugins: [ "@elizaos/plugin-anthropic", "@elizaos/plugin-openai", + "@elizaos/plugin-elevenlabs", // "@elizaos/plugin-local-ai", "@elizaos/plugin-discord", "@elizaos/plugin-node", @@ -535,4 +536,4 @@ export const defaultCharacter: Character = { "provocative", ], extends: [], -}; \ No newline at end of file +}; diff --git a/packages/plugin-elevenlabs/.gitignore b/packages/plugin-elevenlabs/.gitignore new file mode 100644 index 0000000000..77738287f0 --- /dev/null +++ b/packages/plugin-elevenlabs/.gitignore @@ -0,0 +1 @@ +dist/ \ No newline at end of file diff --git a/packages/plugin-elevenlabs/.npmignore b/packages/plugin-elevenlabs/.npmignore new file mode 100644 index 0000000000..078562ecea --- /dev/null +++ b/packages/plugin-elevenlabs/.npmignore @@ -0,0 +1,6 @@ +* + +!dist/** +!package.json +!readme.md +!tsup.config.ts \ No newline at end of file diff --git a/packages/plugin-elevenlabs/README.md b/packages/plugin-elevenlabs/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/plugin-elevenlabs/biome.json b/packages/plugin-elevenlabs/biome.json new file mode 100644 index 0000000000..d7e0349301 --- /dev/null +++ b/packages/plugin-elevenlabs/biome.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.5.3/schema.json", + "extends": ["../../biome.json"], + "organizeImports": { + "enabled": false + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 4, + "lineWidth": 100 + }, + "javascript": { + "formatter": { + "quoteStyle": "single" + } + }, + "files": { + "ignore": [ + "dist/**/*", + "extra/**/*", + "node_modules/**/*" + ] + } +} \ No newline at end of file diff --git a/packages/plugin-elevenlabs/package.json b/packages/plugin-elevenlabs/package.json new file mode 100644 index 0000000000..a3d83e3add --- /dev/null +++ b/packages/plugin-elevenlabs/package.json @@ -0,0 +1,73 @@ +{ + "name": "@elizaos/plugin-elevenlabs", + "version": "1.0.0-alpha.0", + "type": "module", + "main": "dist/index.js", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } + }, + "files": [ + "dist" + ], + "dependencies": { + "@elizaos/core": "workspace:*", + "tsup": "8.3.5" + }, + "scripts": { + "build": "tsup --format esm --dts", + "dev": "tsup --format esm --dts --watch", + "test": "vitest run" + }, + "publishConfig": { + "access": "public" + }, + "agentConfig": { + "pluginType": "elizaos:plugin:1.0.0", + "pluginParameters": { + "ELEVENLABS_XI_API_KEY": { + "type": "string", + "description": "API key for ElevenLabs." + }, + "ELEVENLABS_VOICE_ID": { + "type": "string", + "description": "Optional. Voice selection ID." + }, + "ELEVENLABS_MODEL_ID": { + "type": "string", + "description": "Optional. Speech model ID." + }, + "ELEVENLABS_VOICE_STABILITY": { + "type": "string", + "description": "Optional. Controls voice stability." + }, + "ELEVENLABS_OPTIMIZE_STREAMING_LATENCY": { + "type": "string", + "description": "Optional. Adjusts streaming latency." + }, + "ELEVENLABS_OUTPUT_FORMAT": { + "type": "string", + "description": "Optional. Output format (e.g., pcm_16000)." + }, + "ELEVENLABS_VOICE_SIMILARITY_BOOST": { + "type": "string", + "description": "Optional. Adjusts similarity to the reference voice (0-1)." + }, + "ELEVENLABS_VOICE_STYLE": { + "type": "string", + "description": "Optional. Controls voice style intensity (0-1)." + }, + "ELEVENLABS_VOICE_USE_SPEAKER_BOOST": { + "type": "string", + "description": "Optional. Enhances speaker presence (true/false)." + } + } + } +} diff --git a/packages/plugin-elevenlabs/src/index.ts b/packages/plugin-elevenlabs/src/index.ts new file mode 100644 index 0000000000..b8dbc0e84d --- /dev/null +++ b/packages/plugin-elevenlabs/src/index.ts @@ -0,0 +1,100 @@ +import { IAgentRuntime, Plugin, logger, ModelClass } from "@elizaos/core"; +import { Readable } from "node:stream"; +import { prependWavHeader } from "./utils.ts"; + +function getVoiceSettings(runtime: IAgentRuntime) { + const getSetting = (key: string, fallback = "") => + process.env[key] || runtime.getSetting(key) || fallback; + + return { + apiKey: getSetting("ELEVENLABS_XI_API_KEY"), + voiceId: getSetting("ELEVENLABS_VOICE_ID"), + model: getSetting("ELEVENLABS_MODEL_ID", "eleven_monolingual_v1"), + stability: getSetting("ELEVENLABS_VOICE_STABILITY", "0.5"), + latency: getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY", "0"), + outputFormat: getSetting("ELEVENLABS_OUTPUT_FORMAT", "pcm_16000"), + similarity: getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST", "0.75"), + style: getSetting("ELEVENLABS_VOICE_STYLE", "0"), + speakerBoost: getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST", "true"), + }; +} + +async function fetchSpeech(runtime: IAgentRuntime, text: string) { + const settings = getVoiceSettings(runtime); + try { + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${settings.voiceId}/stream?optimize_streaming_latency=${settings.latency}&output_format=${settings.outputFormat}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "xi-api-key": settings.apiKey, + }, + body: JSON.stringify({ + model_id: settings.model, + text, + voice_settings: { + similarity_boost: settings.similarity, + stability: settings.stability, + style: settings.style, + use_speaker_boost: settings.speakerBoost, + }, + }), + } + ); + if (response.status !== 200) { + const errorBodyString = await response.text(); + const errorBody = JSON.parse(errorBodyString); + + if (response.status === 401 && errorBody.detail?.status === "quota_exceeded") { + logger.log("ElevenLabs quota exceeded"); + throw new Error("QUOTA_EXCEEDED"); + } + throw new Error(`Received status ${response.status} from Eleven Labs API: ${JSON.stringify(errorBody)}`); + } + return Readable.fromWeb(response.body); + } catch (error) { + logger.error(error); + return new Readable({ read() {} }); + } +} + +export const elevenLabsPlugin: Plugin = { + name: "elevenLabs", + description: "ElevenLabs plugin", + models: { + [ModelClass.TEXT_TO_SPEECH]: async (runtime, text) => { + const stream = await fetchSpeech(runtime, text); + return getVoiceSettings(runtime).outputFormat.startsWith("pcm_") + ? prependWavHeader(stream, 1024 * 1024 * 100, parseInt(getVoiceSettings(runtime).outputFormat.slice(4)), 1, 16) + : stream; + }, + }, + tests: [ + { + name: "test eleven labs", + tests: [ + { + name: "Eleven Labs API key validation", + fn: async (runtime: IAgentRuntime) => { + if (!getVoiceSettings(runtime).apiKey) { + throw new Error("Missing API key: Please provide a valid Eleven Labs API key."); + } + }, + }, + { + name: "Eleven Labs API response", + fn: async (runtime: IAgentRuntime) => { + try { + await fetchSpeech(runtime, "test"); + } catch(error) { + throw new Error(`Failed to fetch speech from Eleven Labs API: ${error.message || "Unknown error occurred"}`); + } + + }, + }, + ], + }, + ], +}; +export default elevenLabsPlugin; diff --git a/packages/plugin-elevenlabs/src/utils.ts b/packages/plugin-elevenlabs/src/utils.ts new file mode 100644 index 0000000000..85fa8b1821 --- /dev/null +++ b/packages/plugin-elevenlabs/src/utils.ts @@ -0,0 +1,53 @@ +import { PassThrough } from "stream"; +import { Readable } from "node:stream"; + +export function getWavHeader( + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Buffer { + const wavHeader = Buffer.alloc(44); + wavHeader.write("RIFF", 0); + wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 + wavHeader.write("WAVE", 8); + wavHeader.write("fmt ", 12); + wavHeader.writeUInt32LE(16, 16); // Length of format data + wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) + wavHeader.writeUInt16LE(channelCount, 22); // Number of channels + wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate + wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate + wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) + wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample + wavHeader.write("data", 36); // Data chunk header + wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size + return wavHeader; +} + +export function prependWavHeader( + readable: Readable, + audioLength: number, + sampleRate: number, + channelCount = 1, + bitsPerSample = 16 +): Readable { + const wavHeader = getWavHeader( + audioLength, + sampleRate, + channelCount, + bitsPerSample + ); + let pushedHeader = false; + const passThrough = new PassThrough(); + readable.on("data", (data) => { + if (!pushedHeader) { + passThrough.push(wavHeader); + pushedHeader = true; + } + passThrough.push(data); + }); + readable.on("end", () => { + passThrough.end(); + }); + return passThrough; +} diff --git a/packages/plugin-elevenlabs/tsconfig.json b/packages/plugin-elevenlabs/tsconfig.json new file mode 100644 index 0000000000..2153cf4134 --- /dev/null +++ b/packages/plugin-elevenlabs/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleResolution": "Bundler", + "strict": false, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": false, + "allowImportingTsExtensions": true, + "declaration": true, + "emitDeclarationOnly": true, + "resolveJsonModule": true, + "noImplicitAny": false, + "allowJs": true, + "checkJs": false, + "noEmitOnError": false, + "moduleDetection": "force", + "allowArbitraryExtensions": true + }, + "include": ["src/**/*.ts"] +} \ No newline at end of file diff --git a/packages/plugin-elevenlabs/tsup.config.ts b/packages/plugin-elevenlabs/tsup.config.ts new file mode 100644 index 0000000000..76f9797e29 --- /dev/null +++ b/packages/plugin-elevenlabs/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + outDir: "dist", + sourcemap: true, + clean: true, + format: ["esm"], + external: [ + ], +}); diff --git a/packages/plugin-elevenlabs/vitest.config.ts b/packages/plugin-elevenlabs/vitest.config.ts new file mode 100644 index 0000000000..adbf725538 --- /dev/null +++ b/packages/plugin-elevenlabs/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + }, +});