From 040ed9cb9875fb9b14afe12ad05001b3e9f2b5d5 Mon Sep 17 00:00:00 2001 From: "TechnoHouse (deephbz)" <13776377+deephbz@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:00:05 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20feat:=20Support=20include=5Frea?= =?UTF-8?q?soning=20for=20OpenRouter=20provider's=20models.=20Solves=20#57?= =?UTF-8?q?66?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/modelProviders/openrouter.ts | 13 +++++++++++++ src/libs/agent-runtime/openrouter/index.ts | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/config/modelProviders/openrouter.ts b/src/config/modelProviders/openrouter.ts index d2b72e437cc59..508b03664d6fc 100644 --- a/src/config/modelProviders/openrouter.ts +++ b/src/config/modelProviders/openrouter.ts @@ -214,6 +214,19 @@ const OpenRouter: ModelProviderCard = { }, releasedAt: '2024-09-05', }, + { + contextWindowTokens: 163_840, + description: 'DeepSeek-R1', + displayName: 'DeepSeek R1', + enabled: true, + functionCall: true, + id: 'deepseek/deepseek-r1', + pricing: { + input: 3, + output: 8, + }, + releasedAt: '2025-01-20', + }, { contextWindowTokens: 131_072, description: diff --git a/src/libs/agent-runtime/openrouter/index.ts b/src/libs/agent-runtime/openrouter/index.ts index 457fb422072ea..a5609fbf92c02 100644 --- a/src/libs/agent-runtime/openrouter/index.ts +++ b/src/libs/agent-runtime/openrouter/index.ts @@ -6,6 +6,14 @@ import { OpenRouterModelCard } from './type'; export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ baseURL: 'https://openrouter.ai/api/v1', + chatCompletion: { + handlePayload: (payload) => { + return { + ...payload, + include_reasoning: true, + } as any; + }, + }, constructorOptions: { defaultHeaders: { 'HTTP-Referer': 'https://chat-preview.lobehub.com', @@ -17,10 +25,7 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ }, models: { transformModel: (m) => { - const visionKeywords = [ - 'qwen/qvq', - 'vision', - ]; + const visionKeywords = ['qwen/qvq', 'vision']; const reasoningKeywords = [ 'deepseek/deepseek-r1', @@ -28,7 +33,7 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ 'openai/o3', 'qwen/qvq', 'qwen/qwq', - 'thinking' + 'thinking', ]; const model = m as unknown as OpenRouterModelCard; @@ -45,11 +50,11 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ typeof model.top_provider.max_completion_tokens === 'number' ? model.top_provider.max_completion_tokens : undefined, - reasoning: reasoningKeywords.some(keyword => model.id.toLowerCase().includes(keyword)), + reasoning: reasoningKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)), vision: model.description.includes('vision') || model.description.includes('multimodal') || - visionKeywords.some(keyword => model.id.toLowerCase().includes(keyword)), + visionKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)), }; }, }, From 6c0a1a3003e0c3e409943f674b3e75441034c06d Mon Sep 17 00:00:00 2001 From: "TechnoHouse (deephbz)" <13776377+deephbz@users.noreply.github.com> Date: Sun, 9 Feb 2025 11:22:56 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20feat:=20Support=20display=20Ope?= =?UTF-8?q?nRouter's=20"reasoning"=20output=20by=20transforming=20its=20"r?= =?UTF-8?q?easoning"=20to=20be=20consistent=20with=20most=20other=20platfo?= =?UTF-8?q?rms:=20reasoning=20outputs=20wrapped=20by=20=20XML=20tag?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent-runtime/openrouter/index.test.ts | 4 +- src/libs/agent-runtime/openrouter/index.ts | 3 + src/libs/agent-runtime/utils/streams/index.ts | 1 + .../utils/streams/openrouter.test.ts | 359 ++++++++++++++++++ .../agent-runtime/utils/streams/openrouter.ts | 82 ++++ 5 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 src/libs/agent-runtime/utils/streams/openrouter.test.ts create mode 100644 src/libs/agent-runtime/utils/streams/openrouter.ts diff --git a/src/libs/agent-runtime/openrouter/index.test.ts b/src/libs/agent-runtime/openrouter/index.test.ts index 4ed3ef03b2924..d837c95b048cb 100644 --- a/src/libs/agent-runtime/openrouter/index.test.ts +++ b/src/libs/agent-runtime/openrouter/index.test.ts @@ -79,14 +79,14 @@ describe('LobeOpenRouterAI', () => { // Assert expect(instance['client'].chat.completions.create).toHaveBeenCalledWith( - { + expect.objectContaining({ max_tokens: 1024, messages: [{ content: 'Hello', role: 'user' }], stream: true, model: 'mistralai/mistral-7b-instruct:free', temperature: 0.7, top_p: 1, - }, + }), { headers: { Accept: '*/*' } }, ); expect(result).toBeInstanceOf(Response); diff --git a/src/libs/agent-runtime/openrouter/index.ts b/src/libs/agent-runtime/openrouter/index.ts index a5609fbf92c02..3e9b051336a0e 100644 --- a/src/libs/agent-runtime/openrouter/index.ts +++ b/src/libs/agent-runtime/openrouter/index.ts @@ -2,6 +2,7 @@ import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders'; import { ModelProvider } from '../types'; import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; +import { OpenRouterReasoningStream } from '../utils/streams'; import { OpenRouterModelCard } from './type'; export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ @@ -11,8 +12,10 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ return { ...payload, include_reasoning: true, + stream: payload.stream ?? true, } as any; }, + handleStream: OpenRouterReasoningStream, }, constructorOptions: { defaultHeaders: { diff --git a/src/libs/agent-runtime/utils/streams/index.ts b/src/libs/agent-runtime/utils/streams/index.ts index 65ea36449bd06..ec7f404f7a79a 100644 --- a/src/libs/agent-runtime/utils/streams/index.ts +++ b/src/libs/agent-runtime/utils/streams/index.ts @@ -4,6 +4,7 @@ export * from './bedrock'; export * from './google-ai'; export * from './ollama'; export * from './openai'; +export * from './openrouter'; export * from './protocol'; export * from './qwen'; export * from './spark'; diff --git a/src/libs/agent-runtime/utils/streams/openrouter.test.ts b/src/libs/agent-runtime/utils/streams/openrouter.test.ts new file mode 100644 index 0000000000000..9deb135f0d9fe --- /dev/null +++ b/src/libs/agent-runtime/utils/streams/openrouter.test.ts @@ -0,0 +1,359 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +import { OpenRouterReasoningStream } from './openrouter'; + +describe('OpenRouterReasoningStream', () => { + beforeAll(() => { + // No-op + }); + + it('should wrap first reasoning chunk with and subsequent reasoning is appended', async () => { + // Suppose we get two reasoning-only chunks, then content + const mockOpenAIStream = new ReadableStream({ + start(controller) { + // Reasoning chunk #1 + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + reasoning: 'Hello', + }, + index: 0, + }, + ], + id: 'test-1', + }); + // Reasoning chunk #2 + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + reasoning: ', world!', + }, + index: 1, + }, + ], + id: 'test-1', + }); + // Content chunk => triggers closing + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + content: 'Now real content', + }, + index: 2, + }, + ], + id: 'test-1', + }); + // Finally a stop chunk + controller.enqueue({ + choices: [ + { + delta: null, + finish_reason: 'stop', + index: 3, + }, + ], + id: 'test-1', + }); + + controller.close(); + }, + }); + + // Mock any callbacks you want to track + const onStartMock = vi.fn(); + const onTextMock = vi.fn(); + const onTokenMock = vi.fn(); + const onCompletionMock = vi.fn(); + + const protocolStream = OpenRouterReasoningStream(mockOpenAIStream, { + onStart: onStartMock, + onText: onTextMock, + onToken: onTokenMock, + onCompletion: onCompletionMock, + }); + + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore - for-await usage on a ReadableStream in Node + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + expect(chunks).toEqual([ + // chunk #1: SSE lines + 'id: test-1\n', + 'event: text\n', + 'data: "Hello"\n\n', + + // chunk #2: SSE lines + 'id: test-1\n', + 'event: text\n', + 'data: ", world!"\n\n', + + // chunk #3: SSE lines => content => closes out + 'id: test-1\n', + 'event: text\n', + 'data: "Now real content"\n\n', + + // chunk #4: SSE lines => stop + 'id: test-1\n', + 'event: stop\n', + 'data: "stop"\n\n', + ]); + + // Verify callbacks were triggered + expect(onStartMock).toHaveBeenCalledTimes(1); // Called once at stream start + expect(onTextMock).toHaveBeenCalledTimes(3); // We got 3 text events + expect(onTokenMock).toHaveBeenCalledTimes(3); // Each text is broken down into tokens if you want + expect(onCompletionMock).toHaveBeenCalledTimes(1); // stop event => completion + }); + + it('should output content immediately if reasoning is empty or absent', async () => { + // This simulates a chunk that only has content from the start + const mockStream = new ReadableStream({ + start(controller) { + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + content: 'No reasoning, just content.', + }, + }, + ], + id: 'test-2', + }); + controller.enqueue({ + choices: [ + { + delta: null, + finish_reason: 'stop', + }, + ], + id: 'test-2', + }); + + controller.close(); + }, + }); + + const protocolStream = OpenRouterReasoningStream(mockStream); + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + // Notice there is no or at all + expect(chunks).toEqual([ + 'id: test-2\n', + 'event: text\n', + 'data: "No reasoning, just content."\n\n', + 'id: test-2\n', + 'event: stop\n', + 'data: "stop"\n\n', + ]); + }); + + it('should handle empty stream with no chunks', async () => { + const mockStream = new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + + const protocolStream = OpenRouterReasoningStream(mockStream); + + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + expect(chunks).toEqual([]); + }); + + it('should handle chunk with no choices', async () => { + const mockStream = new ReadableStream({ + start(controller) { + controller.enqueue({ + choices: [], + id: 'test-3', + }); + controller.close(); + }, + }); + + const protocolStream = OpenRouterReasoningStream(mockStream); + + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + // This means the transform function sees no choice => fallback is "data" event + expect(chunks).toEqual([ + 'id: test-3\n', + 'event: data\n', + 'data: {"choices":[],"id":"test-3"}\n\n', + ]); + }); + + it('should handle consecutive reasoning-only chunks with no content', async () => { + const mockStream = new ReadableStream({ + start(controller) { + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + reasoning: 'Just reasoning first chunk', + }, + }, + ], + id: 'test-4', + }); + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + reasoning: 'Continuing reasoning second chunk', + }, + }, + ], + id: 'test-4', + }); + // finish + controller.enqueue({ + choices: [ + { + delta: null, + finish_reason: 'stop', + }, + ], + id: 'test-4', + }); + controller.close(); + }, + }); + + const protocolStream = OpenRouterReasoningStream(mockStream); + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + // No content arrived => never closed the tag + expect(chunks).toEqual([ + 'id: test-4\n', + 'event: text\n', + 'data: "Just reasoning first chunk"\n\n', + + 'id: test-4\n', + 'event: text\n', + 'data: "Continuing reasoning second chunk"\n\n', + + 'id: test-4\n', + 'event: stop\n', + 'data: "stop"\n\n', + ]); + }); + + it('should handle reasonings, then partial content, then more content', async () => { + const mockStream = new ReadableStream({ + start(controller) { + // Reasoning chunk + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + reasoning: 'I am thinking step #1...', + }, + }, + ], + id: 'test-5', + }); + // Content chunk #1 => closes out + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + content: 'Partial answer. ', + }, + }, + ], + id: 'test-5', + }); + // Content chunk #2 => appended, no new + controller.enqueue({ + choices: [ + { + delta: { + role: 'assistant', + content: 'Second part of answer.', + }, + }, + ], + id: 'test-5', + }); + // Stop chunk + controller.enqueue({ + choices: [ + { + delta: null, + finish_reason: 'stop', + }, + ], + id: 'test-5', + }); + controller.close(); + }, + }); + + const protocolStream = OpenRouterReasoningStream(mockStream); + const decoder = new TextDecoder(); + const chunks: string[] = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + // Notice how only the first content chunk triggers "" + expect(chunks).toEqual([ + 'id: test-5\n', + 'event: text\n', + 'data: "I am thinking step #1..."\n\n', + 'id: test-5\n', + 'event: text\n', + 'data: "Partial answer. "\n\n', + 'id: test-5\n', + 'event: text\n', + 'data: "Second part of answer."\n\n', + 'id: test-5\n', + 'event: stop\n', + 'data: "stop"\n\n', + ]); + }); +}); diff --git a/src/libs/agent-runtime/utils/streams/openrouter.ts b/src/libs/agent-runtime/utils/streams/openrouter.ts new file mode 100644 index 0000000000000..777d479fde8e3 --- /dev/null +++ b/src/libs/agent-runtime/utils/streams/openrouter.ts @@ -0,0 +1,82 @@ +import OpenAI from 'openai'; +import type { Stream } from 'openai/streaming'; + +import { ChatStreamCallbacks } from '../../types'; +import { + StreamProtocolChunk, + convertIterableToStream, + createCallbacksTransformer, + createSSEProtocolTransformer, +} from './protocol'; + +/** + * Create a closure to track whether we’ve inserted `` and/or closed it. + */ +function createOpenRouterReasoningTransformer() { + let reasoningStarted = false; + let contentStarted = false; + let insertedThink = false; + + return function transformOpenRouterChunk(chunk: OpenAI.ChatCompletionChunk): StreamProtocolChunk { + const item = chunk.choices?.[0]; + if (!item) { + return { data: chunk, id: chunk.id, type: 'data' }; + } + + // 1) If we have a finish_reason, treat it as stop. + if (item.finish_reason) { + return { data: item.finish_reason, id: chunk.id, type: 'stop' }; + } + + // 2) Then handle any delta + const { content, reasoning } = (item.delta as { content?: string; reasoning?: string }) || {}; + + const isContentNonEmpty = typeof content === 'string' && content.length > 0; + const isReasoningNonEmpty = typeof reasoning === 'string' && reasoning.length > 0; + + // 3) Reasoning logic + if (!contentStarted && isReasoningNonEmpty) { + if (!reasoningStarted) reasoningStarted = true; + if (!insertedThink) { + insertedThink = true; + return { data: `${reasoning}`, id: chunk.id, type: 'text' }; + } + return { data: reasoning, id: chunk.id, type: 'text' }; + } + + // 4) Content logic + if (isContentNonEmpty) { + if (!contentStarted) { + contentStarted = true; + if (reasoningStarted && insertedThink) { + return { data: `${content}`, id: chunk.id, type: 'text' }; + } + } + return { data: content, id: chunk.id, type: 'text' }; + } + + // 5) Fallback + return { data: chunk, id: chunk.id, type: 'data' }; + }; +} + +/** + * The main stream entry point for OpenRouter, similar to Qwen’s “QwenAIStream.” + */ +export function OpenRouterReasoningStream( + stream: Stream | ReadableStream, + callbacks?: ChatStreamCallbacks, +) { + // Convert the stream if it’s an AsyncIterable + const readableStream = + stream instanceof ReadableStream ? stream : convertIterableToStream(stream); + + // Create our chunk-by-chunk transformer + const transformFn = createOpenRouterReasoningTransformer(); + + // 1. Transform each chunk to a standard SSE protocol event + // 2. Pipe it through the user’s callback hooks + return readableStream + .pipeThrough(createSSEProtocolTransformer(transformFn)) + .pipeThrough(createCallbacksTransformer(callbacks)); +} From 109dac810da41322c247e4c2f71e970b265a51ce Mon Sep 17 00:00:00 2001 From: "TechnoHouse (deephbz)" <13776377+deephbz@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:02:38 +0800 Subject: [PATCH 3/5] Revert openrouter stream transformer: Capture reasoning content inside openai stream processor --- src/libs/agent-runtime/openrouter/index.ts | 2 - src/libs/agent-runtime/utils/streams/index.ts | 1 - .../agent-runtime/utils/streams/openai.ts | 8 +- .../utils/streams/openrouter.test.ts | 359 ------------------ .../agent-runtime/utils/streams/openrouter.ts | 82 ---- 5 files changed, 6 insertions(+), 446 deletions(-) delete mode 100644 src/libs/agent-runtime/utils/streams/openrouter.test.ts delete mode 100644 src/libs/agent-runtime/utils/streams/openrouter.ts diff --git a/src/libs/agent-runtime/openrouter/index.ts b/src/libs/agent-runtime/openrouter/index.ts index 3e9b051336a0e..2767ca16c0306 100644 --- a/src/libs/agent-runtime/openrouter/index.ts +++ b/src/libs/agent-runtime/openrouter/index.ts @@ -2,7 +2,6 @@ import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders'; import { ModelProvider } from '../types'; import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; -import { OpenRouterReasoningStream } from '../utils/streams'; import { OpenRouterModelCard } from './type'; export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ @@ -15,7 +14,6 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({ stream: payload.stream ?? true, } as any; }, - handleStream: OpenRouterReasoningStream, }, constructorOptions: { defaultHeaders: { diff --git a/src/libs/agent-runtime/utils/streams/index.ts b/src/libs/agent-runtime/utils/streams/index.ts index ec7f404f7a79a..65ea36449bd06 100644 --- a/src/libs/agent-runtime/utils/streams/index.ts +++ b/src/libs/agent-runtime/utils/streams/index.ts @@ -4,7 +4,6 @@ export * from './bedrock'; export * from './google-ai'; export * from './ollama'; export * from './openai'; -export * from './openrouter'; export * from './protocol'; export * from './qwen'; export * from './spark'; diff --git a/src/libs/agent-runtime/utils/streams/openai.ts b/src/libs/agent-runtime/utils/streams/openai.ts index 1a1124875e4b1..f9f354ee16d8f 100644 --- a/src/libs/agent-runtime/utils/streams/openai.ts +++ b/src/libs/agent-runtime/utils/streams/openai.ts @@ -88,8 +88,12 @@ export const transformOpenAIStream = ( } if (item.delta) { - let reasoning_content = - 'reasoning_content' in item.delta ? item.delta.reasoning_content : null; + let reasoning_content = (() => { + if ('reasoning_content' in item.delta) return item.delta.reasoning_content; + if ('reasoning' in item.delta) return item.delta.reasoning; + return null; + })(); + let content = 'content' in item.delta ? item.delta.content : null; // DeepSeek reasoner will put thinking in the reasoning_content field diff --git a/src/libs/agent-runtime/utils/streams/openrouter.test.ts b/src/libs/agent-runtime/utils/streams/openrouter.test.ts deleted file mode 100644 index 9deb135f0d9fe..0000000000000 --- a/src/libs/agent-runtime/utils/streams/openrouter.test.ts +++ /dev/null @@ -1,359 +0,0 @@ -import { beforeAll, describe, expect, it, vi } from 'vitest'; - -import { OpenRouterReasoningStream } from './openrouter'; - -describe('OpenRouterReasoningStream', () => { - beforeAll(() => { - // No-op - }); - - it('should wrap first reasoning chunk with and subsequent reasoning is appended', async () => { - // Suppose we get two reasoning-only chunks, then content - const mockOpenAIStream = new ReadableStream({ - start(controller) { - // Reasoning chunk #1 - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - reasoning: 'Hello', - }, - index: 0, - }, - ], - id: 'test-1', - }); - // Reasoning chunk #2 - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - reasoning: ', world!', - }, - index: 1, - }, - ], - id: 'test-1', - }); - // Content chunk => triggers closing - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - content: 'Now real content', - }, - index: 2, - }, - ], - id: 'test-1', - }); - // Finally a stop chunk - controller.enqueue({ - choices: [ - { - delta: null, - finish_reason: 'stop', - index: 3, - }, - ], - id: 'test-1', - }); - - controller.close(); - }, - }); - - // Mock any callbacks you want to track - const onStartMock = vi.fn(); - const onTextMock = vi.fn(); - const onTokenMock = vi.fn(); - const onCompletionMock = vi.fn(); - - const protocolStream = OpenRouterReasoningStream(mockOpenAIStream, { - onStart: onStartMock, - onText: onTextMock, - onToken: onTokenMock, - onCompletion: onCompletionMock, - }); - - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for-await usage on a ReadableStream in Node - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - expect(chunks).toEqual([ - // chunk #1: SSE lines - 'id: test-1\n', - 'event: text\n', - 'data: "Hello"\n\n', - - // chunk #2: SSE lines - 'id: test-1\n', - 'event: text\n', - 'data: ", world!"\n\n', - - // chunk #3: SSE lines => content => closes out - 'id: test-1\n', - 'event: text\n', - 'data: "Now real content"\n\n', - - // chunk #4: SSE lines => stop - 'id: test-1\n', - 'event: stop\n', - 'data: "stop"\n\n', - ]); - - // Verify callbacks were triggered - expect(onStartMock).toHaveBeenCalledTimes(1); // Called once at stream start - expect(onTextMock).toHaveBeenCalledTimes(3); // We got 3 text events - expect(onTokenMock).toHaveBeenCalledTimes(3); // Each text is broken down into tokens if you want - expect(onCompletionMock).toHaveBeenCalledTimes(1); // stop event => completion - }); - - it('should output content immediately if reasoning is empty or absent', async () => { - // This simulates a chunk that only has content from the start - const mockStream = new ReadableStream({ - start(controller) { - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - content: 'No reasoning, just content.', - }, - }, - ], - id: 'test-2', - }); - controller.enqueue({ - choices: [ - { - delta: null, - finish_reason: 'stop', - }, - ], - id: 'test-2', - }); - - controller.close(); - }, - }); - - const protocolStream = OpenRouterReasoningStream(mockStream); - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - // Notice there is no or at all - expect(chunks).toEqual([ - 'id: test-2\n', - 'event: text\n', - 'data: "No reasoning, just content."\n\n', - 'id: test-2\n', - 'event: stop\n', - 'data: "stop"\n\n', - ]); - }); - - it('should handle empty stream with no chunks', async () => { - const mockStream = new ReadableStream({ - start(controller) { - controller.close(); - }, - }); - - const protocolStream = OpenRouterReasoningStream(mockStream); - - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - expect(chunks).toEqual([]); - }); - - it('should handle chunk with no choices', async () => { - const mockStream = new ReadableStream({ - start(controller) { - controller.enqueue({ - choices: [], - id: 'test-3', - }); - controller.close(); - }, - }); - - const protocolStream = OpenRouterReasoningStream(mockStream); - - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - // This means the transform function sees no choice => fallback is "data" event - expect(chunks).toEqual([ - 'id: test-3\n', - 'event: data\n', - 'data: {"choices":[],"id":"test-3"}\n\n', - ]); - }); - - it('should handle consecutive reasoning-only chunks with no content', async () => { - const mockStream = new ReadableStream({ - start(controller) { - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - reasoning: 'Just reasoning first chunk', - }, - }, - ], - id: 'test-4', - }); - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - reasoning: 'Continuing reasoning second chunk', - }, - }, - ], - id: 'test-4', - }); - // finish - controller.enqueue({ - choices: [ - { - delta: null, - finish_reason: 'stop', - }, - ], - id: 'test-4', - }); - controller.close(); - }, - }); - - const protocolStream = OpenRouterReasoningStream(mockStream); - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - // No content arrived => never closed the tag - expect(chunks).toEqual([ - 'id: test-4\n', - 'event: text\n', - 'data: "Just reasoning first chunk"\n\n', - - 'id: test-4\n', - 'event: text\n', - 'data: "Continuing reasoning second chunk"\n\n', - - 'id: test-4\n', - 'event: stop\n', - 'data: "stop"\n\n', - ]); - }); - - it('should handle reasonings, then partial content, then more content', async () => { - const mockStream = new ReadableStream({ - start(controller) { - // Reasoning chunk - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - reasoning: 'I am thinking step #1...', - }, - }, - ], - id: 'test-5', - }); - // Content chunk #1 => closes out - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - content: 'Partial answer. ', - }, - }, - ], - id: 'test-5', - }); - // Content chunk #2 => appended, no new - controller.enqueue({ - choices: [ - { - delta: { - role: 'assistant', - content: 'Second part of answer.', - }, - }, - ], - id: 'test-5', - }); - // Stop chunk - controller.enqueue({ - choices: [ - { - delta: null, - finish_reason: 'stop', - }, - ], - id: 'test-5', - }); - controller.close(); - }, - }); - - const protocolStream = OpenRouterReasoningStream(mockStream); - const decoder = new TextDecoder(); - const chunks: string[] = []; - - // @ts-ignore - for await (const chunk of protocolStream) { - chunks.push(decoder.decode(chunk, { stream: true })); - } - - // Notice how only the first content chunk triggers "" - expect(chunks).toEqual([ - 'id: test-5\n', - 'event: text\n', - 'data: "I am thinking step #1..."\n\n', - 'id: test-5\n', - 'event: text\n', - 'data: "Partial answer. "\n\n', - 'id: test-5\n', - 'event: text\n', - 'data: "Second part of answer."\n\n', - 'id: test-5\n', - 'event: stop\n', - 'data: "stop"\n\n', - ]); - }); -}); diff --git a/src/libs/agent-runtime/utils/streams/openrouter.ts b/src/libs/agent-runtime/utils/streams/openrouter.ts deleted file mode 100644 index 777d479fde8e3..0000000000000 --- a/src/libs/agent-runtime/utils/streams/openrouter.ts +++ /dev/null @@ -1,82 +0,0 @@ -import OpenAI from 'openai'; -import type { Stream } from 'openai/streaming'; - -import { ChatStreamCallbacks } from '../../types'; -import { - StreamProtocolChunk, - convertIterableToStream, - createCallbacksTransformer, - createSSEProtocolTransformer, -} from './protocol'; - -/** - * Create a closure to track whether we’ve inserted `` and/or closed it. - */ -function createOpenRouterReasoningTransformer() { - let reasoningStarted = false; - let contentStarted = false; - let insertedThink = false; - - return function transformOpenRouterChunk(chunk: OpenAI.ChatCompletionChunk): StreamProtocolChunk { - const item = chunk.choices?.[0]; - if (!item) { - return { data: chunk, id: chunk.id, type: 'data' }; - } - - // 1) If we have a finish_reason, treat it as stop. - if (item.finish_reason) { - return { data: item.finish_reason, id: chunk.id, type: 'stop' }; - } - - // 2) Then handle any delta - const { content, reasoning } = (item.delta as { content?: string; reasoning?: string }) || {}; - - const isContentNonEmpty = typeof content === 'string' && content.length > 0; - const isReasoningNonEmpty = typeof reasoning === 'string' && reasoning.length > 0; - - // 3) Reasoning logic - if (!contentStarted && isReasoningNonEmpty) { - if (!reasoningStarted) reasoningStarted = true; - if (!insertedThink) { - insertedThink = true; - return { data: `${reasoning}`, id: chunk.id, type: 'text' }; - } - return { data: reasoning, id: chunk.id, type: 'text' }; - } - - // 4) Content logic - if (isContentNonEmpty) { - if (!contentStarted) { - contentStarted = true; - if (reasoningStarted && insertedThink) { - return { data: `${content}`, id: chunk.id, type: 'text' }; - } - } - return { data: content, id: chunk.id, type: 'text' }; - } - - // 5) Fallback - return { data: chunk, id: chunk.id, type: 'data' }; - }; -} - -/** - * The main stream entry point for OpenRouter, similar to Qwen’s “QwenAIStream.” - */ -export function OpenRouterReasoningStream( - stream: Stream | ReadableStream, - callbacks?: ChatStreamCallbacks, -) { - // Convert the stream if it’s an AsyncIterable - const readableStream = - stream instanceof ReadableStream ? stream : convertIterableToStream(stream); - - // Create our chunk-by-chunk transformer - const transformFn = createOpenRouterReasoningTransformer(); - - // 1. Transform each chunk to a standard SSE protocol event - // 2. Pipe it through the user’s callback hooks - return readableStream - .pipeThrough(createSSEProtocolTransformer(transformFn)) - .pipeThrough(createCallbacksTransformer(callbacks)); -} From 88e822c731bdfa9662d4a773db48487ec3c893ca Mon Sep 17 00:00:00 2001 From: "TechnoHouse (deephbz)" <13776377+deephbz@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:03:07 +0800 Subject: [PATCH 4/5] Address PR comments: r1 does not support functional calling --- src/config/modelProviders/openrouter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/modelProviders/openrouter.ts b/src/config/modelProviders/openrouter.ts index 508b03664d6fc..4c1173b08ce5b 100644 --- a/src/config/modelProviders/openrouter.ts +++ b/src/config/modelProviders/openrouter.ts @@ -219,7 +219,7 @@ const OpenRouter: ModelProviderCard = { description: 'DeepSeek-R1', displayName: 'DeepSeek R1', enabled: true, - functionCall: true, + functionCall: false, id: 'deepseek/deepseek-r1', pricing: { input: 3, From d014d346d459b5db1628ec20b83285e1e9da5090 Mon Sep 17 00:00:00 2001 From: "TechnoHouse (deephbz)" <13776377+deephbz@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:13:42 +0800 Subject: [PATCH 5/5] add unittest for openrouter reasoning stream handling --- .../utils/streams/openai.test.ts | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/src/libs/agent-runtime/utils/streams/openai.test.ts b/src/libs/agent-runtime/utils/streams/openai.test.ts index 14629cfe601bf..8d3c2f77aea32 100644 --- a/src/libs/agent-runtime/utils/streams/openai.test.ts +++ b/src/libs/agent-runtime/utils/streams/openai.test.ts @@ -1375,5 +1375,206 @@ describe('OpenAIStream', () => { ].map((i) => `${i}\n`), ); }); + + it('should handle reasoning key from OpenRouter response', async () => { + const data = [ + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { role: 'assistant', reasoning: '' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { reasoning: '您好' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { reasoning: '！' }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '你好', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '很高兴', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '为您', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '提供', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '帮助。', reasoning: null }, + logprobs: null, + finish_reason: null, + }, + ], + }, + { + id: '1', + object: 'chat.completion.chunk', + created: 1737563070, + model: 'deepseek-reasoner', + system_fingerprint: 'fp_1c5d8833bc', + choices: [ + { + index: 0, + delta: { content: '', reasoning: null }, + logprobs: null, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 6, + completion_tokens: 104, + total_tokens: 110, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 70 }, + prompt_cache_hit_tokens: 0, + prompt_cache_miss_tokens: 6, + }, + }, + ]; + + const mockOpenAIStream = new ReadableStream({ + start(controller) { + data.forEach((chunk) => { + controller.enqueue(chunk); + }); + + controller.close(); + }, + }); + + const protocolStream = OpenAIStream(mockOpenAIStream); + + const decoder = new TextDecoder(); + const chunks = []; + + // @ts-ignore + for await (const chunk of protocolStream) { + chunks.push(decoder.decode(chunk, { stream: true })); + } + + expect(chunks).toEqual( + [ + 'id: 1', + 'event: reasoning', + `data: ""\n`, + 'id: 1', + 'event: reasoning', + `data: "您好"\n`, + 'id: 1', + 'event: reasoning', + `data: "！"\n`, + 'id: 1', + 'event: text', + `data: "你好"\n`, + 'id: 1', + 'event: text', + `data: "很高兴"\n`, + 'id: 1', + 'event: text', + `data: "为您"\n`, + 'id: 1', + 'event: text', + `data: "提供"\n`, + 'id: 1', + 'event: text', + `data: "帮助。"\n`, + 'id: 1', + 'event: stop', + `data: "stop"\n`, + ].map((i) => `${i}\n`), + ); + }); }); });