feat(job): send a notification when job fails

ee/tabby-ui/app/(dashboard)/(logs)/jobs/components/job-detail.tsx

@@ -37,6 +37,14 @@ export default function JobRunDetail() {
     (stateLabel === 'Pending' || stateLabel === 'Running') &&
     !currentNode?.stdout
 
+  const handleBackNavigation = () => {
+    if (typeof window !== 'undefined' && window.history.length <= 1) {
+      router.push('/jobs')
+    } else {
+      router.back()
+    }
+  }
+
   React.useEffect(() => {
     let timer: number
     if (currentNode?.createdAt && !currentNode?.finishedAt) {
@@ -61,7 +69,7 @@ export default function JobRunDetail() {
           {currentNode && (
             <>
               <div
-                onClick={() => router.back()}
+                onClick={handleBackNavigation}
                 className="-ml-1 flex cursor-pointer items-center transition-opacity hover:opacity-60"
               >
                 <IconChevronLeft className="mr-1 h-6 w-6" />

ee/tabby-ui/app/globals.css

@@ -103,3 +103,7 @@
 .dialog-without-close-btn > button {
   display: none;
 }
+
+.unread-notification::before {
+  @apply content-[''] float-left w-2 h-2 mr-1.5 mt-2 rounded-full bg-red-400;
+}

ee/tabby-ui/components/notification-box.tsx

@@ -12,18 +12,18 @@ import { useMutation } from '@/lib/tabby/gql'
 import { notificationsQuery } from '@/lib/tabby/query'
 import { ArrayElementType } from '@/lib/types'
 import { cn } from '@/lib/utils'
-
-import LoadingWrapper from './loading-wrapper'
-import { ListSkeleton } from './skeleton'
-import { Button } from './ui/button'
+import { Button } from '@/components/ui/button'
 import {
   DropdownMenu,
   DropdownMenuContent,
   DropdownMenuTrigger
-} from './ui/dropdown-menu'
-import { IconBell, IconCheck } from './ui/icons'
-import { Separator } from './ui/separator'
-import { Tabs, TabsList, TabsTrigger } from './ui/tabs'
+} from '@/components/ui/dropdown-menu'
+import { IconBell, IconCheck } from '@/components/ui/icons'
+import { Separator } from '@/components/ui/separator'
+import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs'
+import LoadingWrapper from '@/components/loading-wrapper'
+import { MemoizedReactMarkdown } from '@/components/markdown'
+import { ListSkeleton } from '@/components/skeleton'
 
 interface Props extends HTMLAttributes<HTMLDivElement> {}
 
@@ -89,7 +89,7 @@ export function NotificationBox({ className, ...rest }: Props) {
           </div>
           <Separator />
           <Tabs
-            className="relative my-2 flex-1 overflow-y-auto px-4"
+            className="relative my-2 flex-1 overflow-y-auto px-5"
             defaultValue="unread"
           >
             <TabsList className="sticky top-0 z-10 grid w-full grid-cols-2">
@@ -156,8 +156,6 @@ interface NotificationItemProps extends HTMLAttributes<HTMLDivElement> {
 }
 
 function NotificationItem({ data }: NotificationItemProps) {
-  const { title, content } = resolveNotification(data.content)
-
   const markNotificationsRead = useMutation(markNotificationsReadMutation)
 
   const onClickMarkRead = () => {
@@ -168,17 +166,14 @@ function NotificationItem({ data }: NotificationItemProps) {
 
   return (
     <div className="space-y-1.5">
-      <div className="space-y-1.5">
-        <div className="flex items-center gap-1.5 overflow-hidden text-sm font-medium">
-          {!data.read && (
-            <span className="h-2 w-2 shrink-0 rounded-full bg-red-400"></span>
-          )}
-          <span className="flex-1 truncate">{title}</span>
-        </div>
-        <div className="whitespace-pre-wrap break-words text-sm text-muted-foreground">
-          {content}
-        </div>
-      </div>
+      <MemoizedReactMarkdown
+        className={cn(
+          'prose max-w-none break-words text-sm dark:prose-invert prose-p:my-1 prose-p:leading-relaxed',
+          { 'unread-notification': !data.read }
+        )}
+      >
+        {data.content}
+      </MemoizedReactMarkdown>
       <div className="flex items-center justify-between text-xs text-muted-foreground">
         <span className="text-muted-foreground">
           {formatNotificationTime(data.createdAt)}

ee/tabby-webserver/src/service/background_job/db.rs

@@ -20,16 +20,19 @@ impl DbMaintainanceJob {
         context: Arc<dyn ContextService>,
         db: DbConn,
     ) -> tabby_schema::Result<()> {
-        let mut errors = vec![];
+        let mut has_error = false;
 
         if let Err(e) = db.delete_expired_token().await {
-            errors.push(format!("Failed to delete expired token: {}", e));
+            has_error = true;
+            logkit::warn!("Failed to delete expired tokens: {}", e);
         };
         if let Err(e) = db.delete_expired_password_resets().await {
-            errors.push(format!("Failed to delete expired password resets: {}", e));
+            has_error = true;
+            logkit::warn!("Failed to delete expired password resets: {}", e);
         };
         if let Err(e) = db.delete_expired_ephemeral_threads().await {
-            errors.push(format!("Failed to delete expired ephemeral threads: {}", e));
+            has_error = true;
+            logkit::warn!("Failed to delete expired ephemeral threads: {}", e);
         };
 
         // Read all active sources
@@ -44,54 +47,57 @@ impl DbMaintainanceJob {
                     .delete_unused_source_id_read_access_policy(&active_source_ids)
                     .await
                 {
-                    errors.push(format!(
+                    has_error = true;
+                    logkit::warn!(
                         "Failed to delete unused source id read access policy: {}",
                         e
-                    ));
+                    );
                 };
             }
             Err(e) => {
-                errors.push(format!("Failed to read active sources: {}", e));
+                has_error = true;
+                logkit::warn!("Failed to read active sources: {}", e);
             }
         }
 
         if let Err(e) = Self::data_retention(now, &db).await {
-            errors.push(format!("Failed to run data retention job: {}", e));
+            has_error = true;
+            logkit::warn!("Failed to run data retention job: {}", e);
         }
 
-        if errors.is_empty() {
+        if !has_error {
             Ok(())
         } else {
             Err(CoreError::Other(anyhow::anyhow!(
-                "Failed to run db maintenance job:\n{}",
-                errors.join(";\n")
+                "Failed to run db maintenance job"
             )))
         }
     }
 
     async fn data_retention(now: DateTime<Utc>, db: &DbConn) -> tabby_schema::Result<()> {
-        let mut errors = vec![];
+        let mut has_error = false;
 
         if let Err(e) = db.delete_job_run_before_three_months(now).await {
-            errors.push(format!(
+            has_error = true;
+            logkit::warn!(
                 "Failed to clean up and retain only the last 3 months of jobs: {}",
                 e
-            ));
+            );
         }
 
         if let Err(e) = db.delete_user_events_before_three_months(now).await {
-            errors.push(format!(
+            has_error = true;
+            logkit::warn!(
                 "Failed to clean up and retain only the last 3 months of user events: {}",
                 e
-            ));
+            );
         }
 
-        if errors.is_empty() {
+        if !has_error {
             Ok(())
         } else {
             Err(CoreError::Other(anyhow::anyhow!(
-                "Failed to run data retention job:\n{}",
-                errors.join(";\n")
+                "Failed to run data retention job"
             )))
         }
     }

ee/tabby-webserver/src/service/background_job/hourly.rs

@@ -35,20 +35,24 @@ impl HourlyJob {
         repository_service: Arc<dyn RepositoryService>,
     ) -> tabby_schema::Result<()> {
         let now = Utc::now();
+        let mut has_error = false;
 
         if let Err(err) = DbMaintainanceJob::cron(now, context_service.clone(), db.clone()).await {
+            has_error = true;
             logkit::warn!("Database maintainance failed: {:?}", err);
         }
 
         if let Err(err) =
             SchedulerGitJob::cron(now, git_repository_service.clone(), job_service.clone()).await
         {
+            has_error = true;
             logkit::warn!("Scheduler job failed: {:?}", err);
         }
 
         if let Err(err) =
             SyncIntegrationJob::cron(now, integration_service.clone(), job_service.clone()).await
         {
+            has_error = true;
             logkit::warn!("Sync integration job failed: {:?}", err);
         }
 
@@ -59,15 +63,24 @@ impl HourlyJob {
         )
         .await
         {
+            has_error = true;
             logkit::warn!("Index issues job failed: {err:?}");
         }
 
         if let Err(err) = IndexGarbageCollection
             .run(repository_service.clone(), context_service.clone())
             .await
         {
+            has_error = true;
             logkit::warn!("Index garbage collection job failed: {err:?}");
         }
-        Ok(())
+
+        if has_error {
+            Err(tabby_schema::CoreError::Other(anyhow::anyhow!(
+                "Hourly job failed"
+            )))
+        } else {
+            Ok(())
+        }
     }
 }

ee/tabby-webserver/src/service/background_job/mod.rs

@@ -28,16 +28,17 @@ use tabby_schema::{
     integration::IntegrationService,
     job::JobService,
     license::LicenseService,
-    notification::NotificationService,
+    notification::{NotificationRecipient, NotificationService},
     repository::{GitRepositoryService, RepositoryService, ThirdPartyRepositoryService},
+    AsID,
 };
 use third_party_integration::SchedulerGithubGitlabJob;
 use tracing::{debug, warn};
 pub use web_crawler::WebCrawlerJob;
 
 use self::third_party_integration::SyncIntegrationJob;
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub enum BackgroundJobEvent {
     SchedulerGitRepository(CodeRepository),
     SchedulerGithubGitlabRepository(ID),
@@ -68,6 +69,48 @@ impl BackgroundJobEvent {
     }
 }
 
+fn background_job_notification_name(event: &BackgroundJobEvent) -> &str {
+    match event {
+        BackgroundJobEvent::SchedulerGitRepository(_) => "Git repository synchronize",
+        BackgroundJobEvent::SchedulerGithubGitlabRepository(_) => {
+            "Github/Gitlab repository synchronize"
+        }
+        BackgroundJobEvent::SyncThirdPartyRepositories(_) => "Integration synchronize",
+        BackgroundJobEvent::WebCrawler(_) => "Web crawler",
+        BackgroundJobEvent::IndexGarbageCollection => "Index garbage collection",
+        BackgroundJobEvent::Hourly => "Hourly",
+        BackgroundJobEvent::Daily => "Daily",
+    }
+}
+
+async fn notify_job_error(
+    notification_service: Arc<dyn NotificationService>,
+    err: &str,
+    event: &BackgroundJobEvent,
+    id: i64,
+) {
+    warn!("job {:?} failed: {:?}", event, err);
+    let name = background_job_notification_name(event);
+    if let Err(err) = notification_service
+        .create(
+            NotificationRecipient::Admin,
+            &format!(
+                r#"Background job failed
+
+Job `{}` has failed.
+
+Please check the log at [Jobs Detail](/jobs/detail?id={}) to identify the underlying issue.
+"#,
+                name,
+                id.as_id()
+            ),
+        )
+        .await
+    {
+        warn!("Failed to send notification: {:?}", err);
+    }
+}
+
 pub async fn start(
     db: DbConn,
     job_service: Arc<dyn JobService>,
@@ -108,6 +151,7 @@ pub async fn start(
                         continue;
                     };
 
+                    let cloned_event = event.clone();
                     if let Err(err) = match event {
                         BackgroundJobEvent::SchedulerGitRepository(repository_config) => {
                             let job = SchedulerGitJob::new(repository_config);
@@ -148,7 +192,8 @@ pub async fn start(
                             ).await
                         }
                     } {
-                        logkit::info!(exit_code = 1; "Job failed {}", err);
+                        logkit::warn!(exit_code = 1; "Job failed: {}", err);
+                        notify_job_error(notification_service.clone(), &err.to_string(), &cloned_event, job.id).await;
                     } else {
                         logkit::info!(exit_code = 0; "Job completed successfully");
                     }

ee/tabby-webserver/src/service/background_job/third_party_integration.rs

@@ -66,6 +66,7 @@ impl SyncIntegrationJob {
                 )
                 .await;
         }
+
         Ok(())
     }
 }

ee/tabby-webserver/src/service/background_job/web_crawler.rs

@@ -14,7 +14,7 @@ use super::helper::Job;
 
 const CRAWLER_TIMEOUT_SECS: u64 = 7200;
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct WebCrawlerJob {
     source_id: String,
     url: String,