Feature add fault tolerance (#81)

* Added ModerationModel ChatMemoryProvider (but might need rework). * Enable the Fault Tolerance Interceptor. Signed-off-by: Emmanuel Hugonnet <[email protected]> --------- Signed-off-by: Emmanuel Hugonnet <[email protected]> Co-authored-by: Buhake Sindi <[email protected]>
smallrye · Dec 19, 2024 · ca625c1 · ca625c1
1 parent b11b956
commit ca625c1
Show file tree

Hide file tree

Showing 17 changed files with 368 additions and 115 deletions.
diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml
@@ -12,7 +12,6 @@
 
     <properties>
         <mainClass>io.helidon.Main</mainClass>
-        <dev.langchain4j.version>0.34.0</dev.langchain4j.version>
     </properties>
 
     <dependencyManagement>

diff --git a/examples/liberty-car-booking/README.md b/examples/liberty-car-booking/README.md
@@ -39,7 +39,7 @@ To package the application in JVM mode run: `mvn package`.
 
 ## Configuration
 
-All configuration is centralized in `microprofile-config.properties`(found is `resources\META-INF` folder) and can be redefined using environment variables.
+All configuration is centralized in `microprofile-config.properties` (found is `resources\META-INF` folder) and can be redefined using environment variables.
 
 ## Running the application
 

diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml
@@ -17,20 +17,23 @@
         </developer>
     </developers>
 
-    <properties>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <project.report.sourceEncoding>UTF-8</project.report.sourceEncoding>
-        <jakartaee-api.version>10.0.0</jakartaee-api.version>
-        <microprofile-api.version>6.1</microprofile-api.version>
-        <compiler-plugin.version>3.13.0</compiler-plugin.version>
-        <war-plugin.version>3.4.0</war-plugin.version>
-        <!--Strictly for OpenLiberty-->
-        <liberty.env.ENGINE_CACHE_DIR>${project.build.directory}/liberty/wlp/usr/shared/resources/lib/</liberty.env.ENGINE_CACHE_DIR>
-    </properties>
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<project.report.sourceEncoding>UTF-8</project.report.sourceEncoding>
+		<jakartaee-api.version>10.0.0</jakartaee-api.version>
+		<microprofile-api.version>6.1</microprofile-api.version>
+		<compiler-plugin.version>3.13.0</compiler-plugin.version>
+		<war-plugin.version>3.4.0</war-plugin.version>
+        <!-- We need only javac to use the release parameter. OpenLiberty doesn't depend on maven-compiler-plugin.' -->
+        <maven.compiler.source combine.self="override" />
+        <maven.compiler.target combine.self="override" />
+
+		<!--Strictly for OpenLiberty-->
+		<liberty.env.ENGINE_CACHE_DIR>${project.build.directory}/liberty/wlp/usr/shared/resources/lib/</liberty.env.ENGINE_CACHE_DIR>
+	</properties>
 
     <dependencyManagement>
         <dependencies>
-            <!-- https://mvnrepository.com/artifact/jakarta.platform/jakarta.jakartaee-api -->
             <dependency>
                 <groupId>jakarta.platform</groupId>
                 <artifactId>jakarta.jakartaee-api</artifactId>
@@ -45,46 +48,19 @@
                 <type>pom</type>
                 <scope>provided</scope>
             </dependency>
-
-            <!--
-            <dependency>
-                    <groupId>io.smallrye.llm.liberty-bundle</groupId>
-                    <artifactId>smallrye-llm-langchain4j-features-bom</artifactId>
-                    <version>1.0.0-SNAPSHOT</version>
-                    <type>pom</type>
-            </dependency>
-            -->
-            <!--
-            <dependency>
-                    <groupId>io.smallrye.llm</groupId>
-                    <artifactId>smallrye-llm-langchain4j-buildcompatible-extension</artifactId>
-                    <version>1.0.0-SNAPSHOT</version>
-            </dependency>
-            -->
-            <!-- https://mvnrepository.com/artifact/ai.djl.huggingface/tokenizers -->
-            <dependency>
-                <groupId>ai.djl.huggingface</groupId>
-                <artifactId>tokenizers</artifactId>
-                <version>0.30.0</version>
-            </dependency>
-        </dependencies>
-    </dependencyManagement>
-
-    <dependencies>
-        <!-- https://mvnrepository.com/artifact/jakarta.platform/jakarta.jakartaee-api -->
-        <!--
-        <dependency>
-                <groupId>jakarta.platform</groupId>
-                <artifactId>jakarta.jakartaee-api</artifactId>
-                <scope>provided</scope>
-        </dependency>
-        -->
-
-        <dependency>
-            <groupId>org.eclipse.microprofile</groupId>
-            <artifactId>microprofile</artifactId>
-            <type>pom</type>
-        </dependency>
+			<dependency>
+			    <groupId>ai.djl.huggingface</groupId>
+			    <artifactId>tokenizers</artifactId>
+			    <version>0.30.0</version>
+			</dependency>
+		</dependencies>
+	</dependencyManagement>
+	<dependencies>
+		<dependency>
+			<groupId>org.eclipse.microprofile</groupId>
+			<artifactId>microprofile</artifactId>
+			<type>pom</type>
+		</dependency>
 
         <dependency>
             <groupId>io.smallrye.llm</groupId>
@@ -96,27 +72,9 @@
             <artifactId>smallrye-llm-langchain4j-portable-extension</artifactId>
         </dependency>
 
-        <!--		
-        <dependency>
-                <groupId>io.smallrye.llm</groupId>
-                <artifactId>smallrye-llm-langchain4j-buildcompatible-extension</artifactId>
-        </dependency>
-        -->
-
-        <!--
-        <dependency>
-                <groupId>io.smallrye.llm.liberty-bundle</groupId>
-                <artifactId>smallrye-llm-langchain4j-feature</artifactId>
-                <version>1.0.0-SNAPSHOT</version>
-                <type>pom</type>
-                <scope>provided</scope>
-        </dependency>
-        -->
-
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
-            <scope>provided</scope>
         </dependency>
 
         <!-- langchain4j dependencies -->
@@ -130,7 +88,6 @@
             <artifactId>langchain4j-hugging-face</artifactId>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/dev.langchain4j/langchain4j-azure-open-ai -->
         <dependency>
             <groupId>dev.langchain4j</groupId>
             <artifactId>langchain4j-azure-open-ai</artifactId>
@@ -146,7 +103,6 @@
             <artifactId>langchain4j-embeddings-all-minilm-l6-v2</artifactId>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/ai.djl.huggingface/tokenizers -->
         <dependency>
             <groupId>ai.djl.huggingface</groupId>
             <artifactId>tokenizers</artifactId>
@@ -156,15 +112,13 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-jdk14</artifactId>
-            <scope>runtime</scope>
         </dependency>
     </dependencies>
 
     <build>
         <finalName>${project.artifactId}</finalName>
         <pluginManagement>
             <plugins>
-                <!-- https://mvnrepository.com/artifact/io.openliberty.tools/liberty-maven-plugin -->
                 <plugin>
                     <groupId>io.openliberty.tools</groupId>
                     <artifactId>liberty-maven-plugin</artifactId>
@@ -197,4 +151,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>
diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java
@@ -40,9 +40,7 @@ public String chatWithAssistant(
     @Path("/fraud")
     public FraudResponse detectFraudForCustomer(
             @QueryParam("name") String name,
-
             @QueryParam("surname") String surname) {
         return fraudService.detectFraudForCustomer(name, surname);
     }
-
 }
diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java
@@ -2,6 +2,8 @@
 
 import java.time.temporal.ChronoUnit;
 
+import jakarta.enterprise.context.ApplicationScoped;
+
 import org.eclipse.microprofile.faulttolerance.Fallback;
 import org.eclipse.microprofile.faulttolerance.Retry;
 import org.eclipse.microprofile.faulttolerance.Timeout;
@@ -10,7 +12,7 @@
 import io.smallrye.llm.spi.RegisterAIService;
 
 //@SuppressWarnings("CdiManagedBeanInconsistencyInspection")
-@RegisterAIService(tools = BookingService.class, chatMemoryMaxMessages = 10)
+@RegisterAIService(scope = ApplicationScoped.class, tools = BookingService.class, chatMemoryMaxMessages = 10)
 public interface ChatAiService {
 
     @SystemMessage("""

diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java
@@ -16,8 +16,10 @@
 import dev.langchain4j.data.document.Document;
 import dev.langchain4j.data.document.parser.TextDocumentParser;
 import dev.langchain4j.data.document.splitter.DocumentSplitters;
+import dev.langchain4j.data.segment.TextSegment;
 import dev.langchain4j.model.embedding.EmbeddingModel;
 import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel;
+import dev.langchain4j.store.embedding.EmbeddingStore;
 import dev.langchain4j.store.embedding.EmbeddingStoreIngestor;
 import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore;
 import lombok.extern.java.Log;
@@ -32,9 +34,7 @@ public class DocRagIngestor {
 
     // Used by ContentRetriever
     @Produces
-    private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>();
-
-    //    private File docs = new File(System.getProperty("docragdir"));
+    private EmbeddingStore<TextSegment> embeddingStore = new InMemoryEmbeddingStore<>();
 
     @Inject
     @ConfigProperty(name = "app.docs-for-rag.dir")
@@ -60,9 +60,4 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl
         log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(),
                 System.currentTimeMillis() - start));
     }
-
-    public static void main(String[] args) {
-
-        System.out.println(InMemoryEmbeddingStore.class.getInterfaces()[0]);
-    }
 }
diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java
@@ -1,14 +1,17 @@
 package io.jefrajames.booking;
 
+import java.time.temporal.ChronoUnit;
+
+import org.eclipse.microprofile.faulttolerance.Fallback;
+import org.eclipse.microprofile.faulttolerance.Retry;
+import org.eclipse.microprofile.faulttolerance.Timeout;
+
 import dev.langchain4j.service.SystemMessage;
 import dev.langchain4j.service.UserMessage;
 import dev.langchain4j.service.V;
 import io.smallrye.llm.spi.RegisterAIService;
 
-@SuppressWarnings("CdiManagedBeanInconsistencyInspection")
-@RegisterAIService(chatMemoryMaxMessages = 5,
-
-        chatLanguageModelName = "chat-model")
+@RegisterAIService(chatMemoryMaxMessages = 5, chatLanguageModelName = "chat-model")
 public interface FraudAiService {
 
     @SystemMessage("""
@@ -44,6 +47,9 @@ A booking overlap (and hence a fraud) occurs when there are several bookings for
 
             You must not wrap JSON response in backticks, markdown, or in any other way, but return it as plain text.
             """)
+    @Timeout(unit = ChronoUnit.MINUTES, value = 5)
+    @Retry(maxRetries = 2)
+    @Fallback(fallbackMethod = "fraudFallback")
     FraudResponse detectFraudForCustomer(@V("name") String name, @V("surname") String surname);
 
     default FraudResponse fraudFallback(String name, String surname) {

diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties
@@ -10,7 +10,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1
 smallrye.llm.plugin.chat-model.config.topP=0.1
 smallrye.llm.plugin.chat-model.config.timeout=120s
 smallrye.llm.plugin.chat-model.config.max-retries=2
-#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false
+#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true
 
 
 smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever

diff --git a/examples/quarkus-car-booking/src/main/resources/application.properties b/examples/quarkus-car-booking/src/main/resources/application.properties
@@ -11,7 +11,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1
 smallrye.llm.plugin.chat-model.config.topP=0.1
 smallrye.llm.plugin.chat-model.config.timeout=120s
 smallrye.llm.plugin.chat-model.config.max-retries=2
-#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false
+#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=false
 
 
 smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever