Skip to content

Commit

Permalink
🐛 fix: fix reasoning output for OpenRouter reasoning models like dee…
Browse files Browse the repository at this point in the history 
…pseek-r1 (lobehub#5903)

* ✨ feat: Support include_reasoning for OpenRouter provider's models. Solves lobehub#5766

* ✨ feat: Support display OpenRouter's "reasoning" output by transforming
its "reasoning" to be consistent with most other platforms: reasoning outputs wrapped by <think> XML tag.

* Revert openrouter stream transformer: Capture reasoning content inside openai stream processor

* Address PR comments: r1 does not support functional calling

* add unittest for openrouter reasoning stream handling
  • Loading branch information
@deephbz @AirboZH
deephbz authored and AirboZH committed Feb 14, 2025
1 parent 505071e commit 2a95c1a
Show file tree
Hide file tree
Showing 5 changed files with 235 additions and 11 deletions.
13 changes: 13 additions & 0 deletions src/config/modelProviders/openrouter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,19 @@ const OpenRouter: ModelProviderCard = {
},
releasedAt: '2024-09-05',
},
{
contextWindowTokens: 163_840,
description: 'DeepSeek-R1',
displayName: 'DeepSeek R1',
enabled: true,
functionCall: false,
id: 'deepseek/deepseek-r1',
pricing: {
input: 3,
output: 8,
},
releasedAt: '2025-01-20',
},
{
contextWindowTokens: 131_072,
description:
Expand Down
4 changes: 2 additions & 2 deletions src/libs/agent-runtime/openrouter/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,14 @@ describe('LobeOpenRouterAI', () => {

// Assert
expect(instance['client'].chat.completions.create).toHaveBeenCalledWith(
{
expect.objectContaining({
max_tokens: 1024,
messages: [{ content: 'Hello', role: 'user' }],
stream: true,
model: 'mistralai/mistral-7b-instruct:free',
temperature: 0.7,
top_p: 1,
},
}),
{ headers: { Accept: '*/*' } },
);
expect(result).toBeInstanceOf(Response);
Expand Down
20 changes: 13 additions & 7 deletions src/libs/agent-runtime/openrouter/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ import { OpenRouterModelCard } from './type';

export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({
baseURL: 'https://openrouter.ai/api/v1',
chatCompletion: {
handlePayload: (payload) => {
return {
...payload,
include_reasoning: true,
stream: payload.stream ?? true,
} as any;
},
},
constructorOptions: {
defaultHeaders: {
'HTTP-Referer': 'https://chat-preview.lobehub.com',
Expand All @@ -17,18 +26,15 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({
},
models: {
transformModel: (m) => {
const visionKeywords = [
'qwen/qvq',
'vision',
];
const visionKeywords = ['qwen/qvq', 'vision'];

const reasoningKeywords = [
'deepseek/deepseek-r1',
'openai/o1',
'openai/o3',
'qwen/qvq',
'qwen/qwq',
'thinking'
'thinking',
];

const model = m as unknown as OpenRouterModelCard;
Expand All @@ -45,11 +51,11 @@ export const LobeOpenRouterAI = LobeOpenAICompatibleFactory({
typeof model.top_provider.max_completion_tokens === 'number'
? model.top_provider.max_completion_tokens
: undefined,
reasoning: reasoningKeywords.some(keyword => model.id.toLowerCase().includes(keyword)),
reasoning: reasoningKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)),
vision:
model.description.includes('vision') ||
model.description.includes('multimodal') ||
visionKeywords.some(keyword => model.id.toLowerCase().includes(keyword)),
visionKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)),
};
},
},
Expand Down
201 changes: 201 additions & 0 deletions src/libs/agent-runtime/utils/streams/openai.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1375,5 +1375,206 @@ describe('OpenAIStream', () => {
].map((i) => `${i}\n`),
);
});

it('should handle reasoning key from OpenRouter response', async () => {
const data = [
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { role: 'assistant', reasoning: '' },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { reasoning: '您好' },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { reasoning: '！' },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '你好', reasoning: null },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '很高兴', reasoning: null },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '为您', reasoning: null },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '提供', reasoning: null },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '帮助。', reasoning: null },
logprobs: null,
finish_reason: null,
},
],
},
{
id: '1',
object: 'chat.completion.chunk',
created: 1737563070,
model: 'deepseek-reasoner',
system_fingerprint: 'fp_1c5d8833bc',
choices: [
{
index: 0,
delta: { content: '', reasoning: null },
logprobs: null,
finish_reason: 'stop',
},
],
usage: {
prompt_tokens: 6,
completion_tokens: 104,
total_tokens: 110,
prompt_tokens_details: { cached_tokens: 0 },
completion_tokens_details: { reasoning_tokens: 70 },
prompt_cache_hit_tokens: 0,
prompt_cache_miss_tokens: 6,
},
},
];

const mockOpenAIStream = new ReadableStream({
start(controller) {
data.forEach((chunk) => {
controller.enqueue(chunk);
});

controller.close();
},
});

const protocolStream = OpenAIStream(mockOpenAIStream);

const decoder = new TextDecoder();
const chunks = [];

// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}

expect(chunks).toEqual(
[
'id: 1',
'event: reasoning',
`data: ""\n`,
'id: 1',
'event: reasoning',
`data: "您好"\n`,
'id: 1',
'event: reasoning',
`data: "！"\n`,
'id: 1',
'event: text',
`data: "你好"\n`,
'id: 1',
'event: text',
`data: "很高兴"\n`,
'id: 1',
'event: text',
`data: "为您"\n`,
'id: 1',
'event: text',
`data: "提供"\n`,
'id: 1',
'event: text',
`data: "帮助。"\n`,
'id: 1',
'event: stop',
`data: "stop"\n`,
].map((i) => `${i}\n`),
);
});
});
});
8 changes: 6 additions & 2 deletions src/libs/agent-runtime/utils/streams/openai.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,12 @@ export const transformOpenAIStream = (
}

if (item.delta) {
let reasoning_content =
'reasoning_content' in item.delta ? item.delta.reasoning_content : null;
let reasoning_content = (() => {
if ('reasoning_content' in item.delta) return item.delta.reasoning_content;
if ('reasoning' in item.delta) return item.delta.reasoning;
return null;
})();

let content = 'content' in item.delta ? item.delta.content : null;

// DeepSeek reasoner will put thinking in the reasoning_content field
Expand Down

0 comments on commit 2a95c1a

Please sign in to comment.