adapt to azure url pattern and do not cache for non-200 response

Signed-off-by: oilbeater <[email protected]>

README.md

@@ -6,6 +6,8 @@ It is fully **CloudFlare Native**: allowing for global scale deployment without
 
 It is written in **TypeScript**: ensuring adaptability to the rapidly evolving AI ecosystem and catering to the diverse needs of application developers.
 
+> Malacca is still an early-stage project, containing many experiments by the author. Currently, it only supports AzureOpenAI. We welcome contributions and ideas from the community to help expand and improve this project.
+
 ## Features
 
 - 🌍 **Global Scale, Zero Maintenance**
@@ -70,6 +72,8 @@ Malacca offers a flexible architecture, allowing users to:
 - Extend or modify existing features.
 - Integrate additional upstream AI API services.
 
+You can just clone and modify the code then `wrangler deploy` to deploy your code globally.
+
 ## Contact
 
 If you have any questions or suggestions, feel free to reach out via email at [[email protected]](mailto:[email protected]).

src/providers/azureOpenAI.ts

@@ -3,15 +3,15 @@ import { AIProvider, AIRequestParams } from '../types';
 import { generateCacheKey } from '../utils/cache';
 import { recordAnalytics } from '../utils/analytics';
 
-const BasePath = '/azure-openai/:resource_name/:deployment_name';
+const BasePath = '/azure-openai/:resource_name/deployments/:deployment_name';
 const ProviderName = 'azure-openai';
 
 const azureOpenAIRoute = new Hono();
 
 azureOpenAIRoute.post('/*', async (c) => {
     const resourceName = c.req.param('resource_name') || '';
     const deploymentName = c.req.param('deployment_name') || '';
-    const functionName = c.req.path.slice(`/azure-openai/${resourceName}/${deploymentName}/`.length);
+    const functionName = c.req.path.slice(`/azure-openai/${resourceName}/deployments/${deploymentName}/`.length);
 
     const params: AIRequestParams = { resourceName, deploymentName, functionName };
     return azureOpenAIProvider.handleRequest(c, params);
@@ -87,7 +87,9 @@ export const azureOpenAIProvider: AIProvider = {
             const duration = Date.now() - startTime;
             recordAnalytics(c, ProviderName, duration, prompt_tokens, completion_tokens);
 
-            c.executionCtx.waitUntil(c.env.MALACCA_CACHE.put(cacheKeyHex, buf));
+            if (response.status === 200) {
+                c.executionCtx.waitUntil(c.env.MALACCA_CACHE.put(cacheKeyHex, buf));
+            }
             await writer.close();
         })();
 

test/index.spec.ts

@@ -20,7 +20,7 @@ describe('Welcome to Malacca worker', () => {
 });
 
 describe('Test Cache', () => {
-	const url = `https://example.com/azure-openai/${import.meta.env.VITE_AZURE_RESOURCE_NAME}/${import.meta.env.VITE_AZURE_DEPLOYMENT_NAME}/chat/completions?api-version=2024-07-01-preview`;
+	const url = `https://example.com/azure-openai/${import.meta.env.VITE_AZURE_RESOURCE_NAME}/deployments/${import.meta.env.VITE_AZURE_DEPLOYMENT_NAME}/chat/completions?api-version=2024-07-01-preview`;
   const createRequestBody = (stream: boolean) => `
   {
     "messages": [
@@ -38,7 +38,7 @@ describe('Test Cache', () => {
         "content": [
           {
             "type": "text",
-            "text": "Tell me a very short story about sunwukong"
+            "text": "Tell me a very short story about Malacca"
           }
         ]
       }
@@ -93,4 +93,34 @@ describe('Test Cache', () => {
     expect(duration/2).toBeGreaterThan(cacheDuration)
   });
 
+  it('should not cache non-200 responses', async () => {
+    const invalidBody = JSON.stringify({
+      messages: [{ role: "user", content: "This is an invalid request" }],
+      stream: "invalid-model",
+      temperature: 0.7,
+      top_p: 0.95,
+      max_tokens: 800,
+    });
+
+    // First request - should return a non-200 response
+    let response = await SELF.fetch(url, {
+      method: 'POST',
+      body: invalidBody,
+      headers: { 'Content-Type': 'application/json', 'api-key': 'oilbeater' }
+    });
+
+    expect(response.status).not.toBe(200);
+    expect(response.headers.get('malacca-cache-status')).toBe('miss');
+
+    // Second request with the same invalid body
+    response = await SELF.fetch(url, {
+      method: 'POST',
+      body: invalidBody,
+      headers: { 'Content-Type': 'application/json', 'api-key': 'oilbeater' }
+    });
+
+    expect(response.status).not.toBe(200);
+    // Should still be a cache miss, as non-200 responses are not cached
+    expect(response.headers.get('malacca-cache-status')).toBe('miss');
+  });
 });