Merge branch 'main' into main

slreznit · Sep 12, 2024 · a7172bf · a7172bf
2 parents 7d2c46c + d2ff7ad
commit a7172bf
Show file tree

Hide file tree

Showing 17 changed files with 228 additions and 102 deletions.
diff --git a/app.py b/app.py
@@ -103,6 +103,7 @@ async def assets(path):
         "show_chat_history_button": app_settings.ui.show_chat_history_button,
     },
     "sanitize_answer": app_settings.base_settings.sanitize_answer,
+    "oyd_enabled": app_settings.base_settings.datasource_type,
 }
 
 

diff --git a/backend/utils.py b/backend/utils.py
@@ -166,10 +166,12 @@ def format_pf_non_streaming_response(
                 "content": chatCompletion[response_field_name] 
             })
         if citations_field_name in chatCompletion:
+            citation_content= {"citations": chatCompletion[citations_field_name]}
             messages.append({ 
                 "role": "tool",
-                "content": chatCompletion[citations_field_name]
+                "content": json.dumps(citation_content)
             })
+
         response_obj = {
             "id": chatCompletion["id"],
             "model": "",

diff --git a/frontend/src/api/models.ts b/frontend/src/api/models.ts
@@ -1,5 +1,5 @@
 export type AskResponse = {
-  answer: string
+  answer: string | []
   citations: Citation[]
   generated_chart: string | null
   error?: string
@@ -40,7 +40,7 @@ export type AzureSqlServerExecResults = {
 export type ChatMessage = {
   id: string
   role: string
-  content: string
+  content: string | [{ type: string; text: string }, { type: string; image_url: { url: string } }]
   end_turn?: boolean
   date: string
   feedback?: Feedback
@@ -138,6 +138,7 @@ export type FrontendSettings = {
   feedback_enabled?: string | null
   ui?: UI
   sanitize_answer?: boolean
+  oyd_enabled?: boolean
 }
 
 export enum Feedback {

diff --git a/frontend/src/components/Answer/Answer.tsx b/frontend/src/components/Answer/Answer.tsx
@@ -247,17 +247,17 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
         <Stack.Item>
           <Stack horizontal grow>
             <Stack.Item grow>
-              <ReactMarkdown
+              {parsedAnswer && <ReactMarkdown
                 linkTarget="_blank"
                 remarkPlugins={[remarkGfm, supersub]}
                 children={
                   SANITIZE_ANSWER
-                    ? DOMPurify.sanitize(parsedAnswer.markdownFormatText, { ALLOWED_TAGS: XSSAllowTags, ALLOWED_ATTR: XSSAllowAttributes })
-                    : parsedAnswer.markdownFormatText
+                    ? DOMPurify.sanitize(parsedAnswer?.markdownFormatText, { ALLOWED_TAGS: XSSAllowTags, ALLOWED_ATTR: XSSAllowAttributes })
+                    : parsedAnswer?.markdownFormatText
                 }
                 className={styles.answerText}
                 components={components}
-              />
+              />}
             </Stack.Item>
             <Stack.Item className={styles.answerHeader}>
               {FEEDBACK_ENABLED && answer.message_id !== undefined && (
@@ -290,15 +290,15 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
             </Stack.Item>
           </Stack>
         </Stack.Item>
-        {parsedAnswer.generated_chart !== null && (
+        {parsedAnswer?.generated_chart !== null && (
           <Stack className={styles.answerContainer}>
             <Stack.Item grow>
-              <img src={`data:image/png;base64, ${parsedAnswer.generated_chart}`} />
+              <img src={`data:image/png;base64, ${parsedAnswer?.generated_chart}`} />
             </Stack.Item>
           </Stack>
         )}
         <Stack horizontal className={styles.answerFooter}>
-          {!!parsedAnswer.citations.length && (
+          {!!parsedAnswer?.citations.length && (
             <Stack.Item onKeyDown={e => (e.key === 'Enter' || e.key === ' ' ? toggleIsRefAccordionOpen() : null)}>
               <Stack style={{ width: '100%' }}>
                 <Stack horizontal horizontalAlign="start" verticalAlign="center">
@@ -352,7 +352,7 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
         </Stack>
         {chevronIsExpanded && (
           <div className={styles.citationWrapper}>
-            {parsedAnswer.citations.map((citation, idx) => {
+            {parsedAnswer?.citations.map((citation, idx) => {
               return (
                 <span
                   title={createCitationFilepath(citation, ++idx)}

diff --git a/frontend/src/components/Answer/AnswerParser.test.ts b/frontend/src/components/Answer/AnswerParser.test.ts
@@ -54,17 +54,3 @@ describe('enumerateCitations', () => {
     expect(results[2].part_index).toEqual(1)
   })
 })
-
-describe('parseAnswer', () => {
-  it('reformats the answer text and reindexes citations', () => {
-    const parsed: ParsedAnswer = parseAnswer(sampleAnswer)
-    expect(parsed.markdownFormatText).toBe('This is an example answer with citations  ^1^  and  ^2^ .')
-    expect(parsed.citations.length).toBe(2)
-    expect(parsed.citations[0].id).toBe('1')
-    expect(parsed.citations[0].reindex_id).toBe('1')
-    expect(parsed.citations[1].id).toBe('2')
-    expect(parsed.citations[1].reindex_id).toBe('2')
-    expect(parsed.citations[0].part_index).toBe(1)
-    expect(parsed.citations[1].part_index).toBe(2)
-  })
-})
diff --git a/frontend/src/components/Answer/AnswerParser.tsx b/frontend/src/components/Answer/AnswerParser.tsx
@@ -6,7 +6,7 @@ export type ParsedAnswer = {
   citations: Citation[]
   markdownFormatText: string,
   generated_chart: string | null
-}
+} | null
 
 export const enumerateCitations = (citations: Citation[]) => {
   const filepathMap = new Map()
@@ -23,6 +23,7 @@ export const enumerateCitations = (citations: Citation[]) => {
 }
 
 export function parseAnswer(answer: AskResponse): ParsedAnswer {
+  if (typeof answer.answer !== "string") return null
   let answerText = answer.answer
   const citationLinks = answerText.match(/\[(doc\d\d?\d?)]/g)
 

diff --git a/frontend/src/components/QuestionInput/QuestionInput.module.css b/frontend/src/components/QuestionInput/QuestionInput.module.css
@@ -62,3 +62,35 @@
     left: 16.5%;
   }
 }
+
+.fileInputContainer {
+  position: absolute;
+  right: 24px;
+  top: 20px;
+}
+
+.fileInput {
+  width: 0;
+  height: 0;
+  opacity: 0;
+  overflow: hidden;
+  position: absolute;
+  z-index: -1;
+}
+
+.fileLabel {
+  display: inline-block;
+  border-radius: 5px;
+  cursor: pointer;
+  text-align: center;
+  font-size: 14px;
+}
+
+.fileIcon {
+  font-size: 20px;
+  color: #424242;
+}
+
+.uploadedImage {
+  margin-right: 70px;
+}
diff --git a/frontend/src/components/QuestionInput/QuestionInput.tsx b/frontend/src/components/QuestionInput/QuestionInput.tsx
@@ -1,13 +1,15 @@
-import { useState } from 'react'
-import { Stack, TextField } from '@fluentui/react'
+import { useContext, useState } from 'react'
+import { FontIcon, Stack, TextField } from '@fluentui/react'
 import { SendRegular } from '@fluentui/react-icons'
 
 import Send from '../../assets/Send.svg'
 
 import styles from './QuestionInput.module.css'
+import { ChatMessage } from '../../api'
+import { AppStateContext } from '../../state/AppProvider'
 
 interface Props {
-  onSend: (question: string, id?: string) => void
+  onSend: (question: ChatMessage['content'], id?: string) => void
   disabled: boolean
   placeholder?: string
   clearOnSend?: boolean
@@ -16,16 +18,46 @@ interface Props {
 
 export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conversationId }: Props) => {
   const [question, setQuestion] = useState<string>('')
+  const [base64Image, setBase64Image] = useState<string | null>(null);
+
+  const appStateContext = useContext(AppStateContext)
+  const OYD_ENABLED = appStateContext?.state.frontendSettings?.oyd_enabled || false;
+
+  const handleImageUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+
+    if (file) {
+      await convertToBase64(file);
+    }
+  };
+
+  const convertToBase64 = async (file: Blob) => {
+    const reader = new FileReader();
+
+    reader.readAsDataURL(file);
+
+    reader.onloadend = () => {
+      setBase64Image(reader.result as string);
+    };
+
+    reader.onerror = (error) => {
+      console.error('Error: ', error);
+    };
+  };
 
   const sendQuestion = () => {
     if (disabled || !question.trim()) {
       return
     }
 
-    if (conversationId) {
-      onSend(question, conversationId)
+    const questionTest: ChatMessage["content"] = base64Image ? [{ type: "text", text: question }, { type: "image_url", image_url: { url: base64Image } }] : question.toString();
+
+    if (conversationId && questionTest !== undefined) {
+      onSend(questionTest, conversationId)
+      setBase64Image(null)
     } else {
-      onSend(question)
+      onSend(questionTest)
+      setBase64Image(null)
     }
 
     if (clearOnSend) {
@@ -58,6 +90,24 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conv
         onChange={onQuestionChange}
         onKeyDown={onEnterPress}
       />
+      {!OYD_ENABLED && (
+        <div className={styles.fileInputContainer}>
+          <input
+            type="file"
+            id="fileInput"
+            onChange={(event) => handleImageUpload(event)}
+            accept="image/*"
+            className={styles.fileInput}
+          />
+          <label htmlFor="fileInput" className={styles.fileLabel} aria-label='Upload Image'>
+            <FontIcon
+              className={styles.fileIcon}
+              iconName={'PhotoCollection'}
+              aria-label='Upload Image'
+            />
+          </label>
+        </div>)}
+      {base64Image && <img className={styles.uploadedImage} src={base64Image} alt="Uploaded Preview" />}
       <div
         className={styles.questionInputSendButtonContainer}
         role="button"

diff --git a/frontend/src/pages/chat/Chat.module.css b/frontend/src/pages/chat/Chat.module.css
@@ -85,6 +85,7 @@
 }
 
 .chatMessageUserMessage {
+  position: relative;
   display: flex;
   padding: 20px;
   background: #edf5fd;
@@ -360,6 +361,15 @@ a {
   cursor: pointer;
 }
 
+.uploadedImageChat {
+  position: absolute;
+  right: -23px;
+  bottom: -35px;
+  max-width: 70%;
+  max-height: 70%;
+  border-radius: 4px;
+}
+
 @media (max-width: 480px) {
   .chatInput {
     width: 90%;
-Original file line number
+Diff line change
@@ Expand Up / @@ -103,6 +103,7 @@ async def assets(path): @@
             "show_chat_history_button": app_settings.ui.show_chat_history_button,
         },
         "sanitize_answer": app_settings.base_settings.sanitize_answer,
+        "oyd_enabled": app_settings.base_settings.datasource_type,
     }
@@ Expand Down @@