From 2a95c1a82af7460cb9bb130b3264175d46751470 Mon Sep 17 00:00:00 2001 From: TechnoHouse <13776377+deephbz@users.noreply.github.com> Date: Wed, 12 Feb 2025 12:47:54 +0800 Subject: [PATCH] =?UTF-8?q?=20=F0=9F=90=9B=20fix:=20fix=20reasoning=20outp?= =?UTF-8?q?ut=20for=20OpenRouter=20reasoning=20models=20like=20deepseek-r1?= =?UTF-8?q?=20(#5903)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat: Support include_reasoning for OpenRouter provider's models. Solves #5766 * ✨ feat: Support display OpenRouter's "reasoning" output by transforming its "reasoning" to be consistent with most other platforms: reasoning outputs wrapped by XML tag. * Revert openrouter stream transformer: Capture reasoning content inside openai stream processor * Address PR comments: r1 does not support functional calling * add unittest for openrouter reasoning stream handling --- src/config/modelProviders/openrouter.ts | 13 ++ .../agent-runtime/openrouter/index.test.ts | 4 +- src/libs/agent-runtime/openrouter/index.ts | 20 +- .../utils/streams/openai.test.ts | 201 ++++++++++++++++++ .../agent-runtime/utils/streams/openai.ts | 8 +- 5 files changed, 235 insertions(+), 11 deletions(-) diff --git a/src/config/modelProviders/openrouter.ts b/src/config/modelProviders/openrouter.ts index d2b72e437cc59..4c1173b08ce5b 100644 --- a/src/config/modelProviders/openrouter.ts +++ b/src/config/modelProviders/openrouter.ts @@ -214,6 +214,19 @@ const OpenRouter: ModelProviderCard = { }, releasedAt: '2024-09-05', }, + { + contextWindowTokens: 163_840, + description: 'DeepSeek-R1', + displayName: 'DeepSeek R1', + enabled: true, + functionCall: false, + id: 'deepseek/deepseek-r1', + pricing: { + input: 3, + output: 8, + }, + releasedAt: '2025-01-20', + }, { contextWindowTokens: 131_072, description: diff --git a/src/libs/agent-runtime/openrouter/index.test.ts b/src/libs/agent-runtime/openrouter/index.test.ts index 4ed3ef03b2924..d837c95b048cb 100644 --- a/src/libs/agent-runtime/openrouter/index.test.ts +++ b/src/libs/agent-runtime/openrouter/index.test.ts @@ -79,14 +79,14 @@ describe('LobeOpenRouterAI', () => { // Assert expect(instance['client'].chat.completions.create).toHaveBeenCalledWith( - { + expect.objectContaining({ max_tokens: 1024, messages: [{ content: 'Hello', role: 'user' }], stream: true, model: 'mistralai/mistral-7b-instruct:free', temperature: 0.7, top_p: 1, - }, + }), { headers: { Accept: '*/*' } }, ); expect(result).toBeInstanceOf(Response); diff --git a/src/libs/agent-runtime/openrouter/index.ts b/src/libs/agent-runtime/openrouter/index.ts index 457fb422072ea..2767ca16c0306 100644 --- a/src/libs/agent-runtime/openrouter/index.ts +++ b/src/libs/agent-runtime/openrouter/index.ts @@ -6,6 +6,15 @@ import { OpenRouterModelCard } from './type'; export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ baseURL: 'https://openrouter.ai/api/v1', + chatCompletion: { + handlePayload: (payload) => { + return { + ...payload, + include_reasoning: true, + stream: payload.stream ?? true, + } as any; + }, + }, constructorOptions: { defaultHeaders: { 'HTTP-Referer': 'https://chat-preview.lobehub.com', @@ -17,10 +26,7 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ }, models: { transformModel: (m) => { - const visionKeywords = [ - 'qwen/qvq', - 'vision', - ]; + const visionKeywords = ['qwen/qvq', 'vision']; const reasoningKeywords = [ 'deepseek/deepseek-r1', @@ -28,7 +34,7 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ 'openai/o3', 'qwen/qvq', 'qwen/qwq', - 'thinking' + 'thinking', ]; const model = m as unknown as OpenRouterModelCard; @@ -45,11 +51,11 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ typeof model.top_provider.max_completion_tokens === 'number' ? model.top_provider.max_completion_tokens : undefined, - reasoning: reasoningKeywords.some(keyword => model.id.toLowerCase().includes(keyword)), + reasoning: reasoningKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)), vision: model.description.includes('vision') || model.description.includes('multimodal') || - visionKeywords.some(keyword => model.id.toLowerCase().includes(keyword)), + visionKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)), }; }, }, diff --git a/src/libs/agent-runtime/utils/streams/openai.test.ts b/src/libs/agent-runtime/utils/streams/openai.test.ts index 14629cfe601bf..8d3c2f77aea32 100644 --- a/src/libs/agent-runtime/utils/streams/openai.test.ts +++ b/src/libs/agent-runtime/utils/streams/openai.test.ts @@ -1375,5 +1375,206 @@ describe('OpenAIStream', () => { ].map((i) => `${i}\n`), ); }); + + it('should handle reasoning key from OpenRouter response', async () => { + const data = [ + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { role: 'assistant', reasoning: '' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { reasoning: '您好' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { reasoning: '！' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '你好', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '很高兴', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '为您', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '提供', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '帮助。', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '', reasoning: null }, + logprobs: null, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 104, + total_tokens: 110, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 70 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 6, + }, + }, + ]; + + const mockOpenAIStream = new ReadableStream({ + start(controller) { + data.forEach((chunk) => { + controller.enqueue(chunk); + }); + + controller.close(); + }, + }); + + const protocolStream = OpenAIStream(mockOpenAIStream); + + const decoder = new TextDecoder(); + const chunks = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + expect(chunks).toEqual( + [ + 'id: 1', + 'event: reasoning', + `data: ""\n`, + 'id: 1', + 'event: reasoning', + `data: "您好"\n`, + 'id: 1', + 'event: reasoning', + `data: "！"\n`, + 'id: 1', + 'event: text', + `data: "你好"\n`, + 'id: 1', + 'event: text', + `data: "很高兴"\n`, + 'id: 1', + 'event: text', + `data: "为您"\n`, + 'id: 1', + 'event: text', + `data: "提供"\n`, + 'id: 1', + 'event: text', + `data: "帮助。"\n`, + 'id: 1', + 'event: stop', + `data: "stop"\n`, + ].map((i) => `${i}\n`), + ); + }); }); }); diff --git a/src/libs/agent-runtime/utils/streams/openai.ts b/src/libs/agent-runtime/utils/streams/openai.ts index 1a1124875e4b1..f9f354ee16d8f 100644 --- a/src/libs/agent-runtime/utils/streams/openai.ts +++ b/src/libs/agent-runtime/utils/streams/openai.ts @@ -88,8 +88,12 @@ export const transformOpenAIStream = ( } if (item.delta) { - let reasoning_content = - 'reasoning_content' in item.delta ? item.delta.reasoning_content : null; + let reasoning_content = (() => { + if ('reasoning_content' in item.delta) return item.delta.reasoning_content; + if ('reasoning' in item.delta) return item.delta.reasoning; + return null; + })(); + let content = 'content' in item.delta ? item.delta.content : null; // DeepSeek reasoner will put thinking in the reasoning_content field