Include mouse data when enhancing prompt

replayio · Jan 30, 2025 · 3c193be · 3c193be
1 parent df8e252
commit 3c193be
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 5 deletions.
diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx
@@ -285,9 +285,7 @@ export const ChatImpl = memo(
       let enhancedPrompt, message;
       try {
         const mouseData = getCurrentMouseData();
-        console.log("MouseData", mouseData);
-
-        enhancedPrompt = await getSimulationEnhancedPrompt(recordingId, messages, userMessage);
+        enhancedPrompt = await getSimulationEnhancedPrompt(recordingId, messages, userMessage, mouseData);
         message = `Explanation of the bug:\n\n${enhancedPrompt}`;
       } catch (e) {
         console.error("Error enhancing prompt", e);

diff --git a/app/lib/replay/SimulationPrompt.ts b/app/lib/replay/SimulationPrompt.ts
@@ -5,6 +5,7 @@ import type { Message } from 'ai';
 import type { SimulationData } from './SimulationData';
 import { SimulationDataVersion } from './SimulationData';
 import { assert, ProtocolClient } from './ReplayProtocolClient';
+import type { MouseData } from './Recording';
 
 export async function getSimulationRecording(
   simulationData: SimulationData,
@@ -56,7 +57,8 @@ Do not describe the specific fix needed.
 export async function getSimulationEnhancedPrompt(
   recordingId: string,
   chatMessages: Message[],
-  userMessage: string
+  userMessage: string,
+  mouseData: MouseData | undefined
 ): Promise<string> {
   const client = new ProtocolClient();
   await client.initialize();
@@ -68,11 +70,16 @@ export async function getSimulationEnhancedPrompt(
       params: { chatId, recordingId },
     });
 
+    let system = SystemPrompt;
+    if (mouseData) {
+      system += `The user pointed to an element on the page <element selector=${JSON.stringify(mouseData.selector)} height=${mouseData.height} width=${mouseData.width} x=${mouseData.x} y=${mouseData.y} />`;
+    }
+
     const messages = [
       {
         role: "system",
         type: "text",
-        content: SystemPrompt,
+        content: system,
       },
       {
         role: "user",
@@ -81,6 +88,8 @@ export async function getSimulationEnhancedPrompt(
       },
     ];
 
+    console.log("ChatSendMessage", messages);
+
     let response: string = "";
     const removeListener = client.listenForMessage("Nut.chatResponsePart", ({ message }: { message: ProtocolMessage }) => {
       console.log("ChatResponsePart", message);