List all issues and commit to index

crates/tabby-scheduler/src/doc/mod.rs

@@ -92,13 +92,35 @@ pub struct DocIndexer {
     indexer: Indexer<SourceDocument>,
 }
 
+pub struct IssueDocument {
+    pub id: String,
+    pub link: String,
+    pub title: String,
+    pub body: String,
+}
+
+impl From<IssueDocument> for SourceDocument {
+    fn from(value: IssueDocument) -> Self {
+        Self {
+            id: value.id,
+            link: value.link,
+            title: value.title,
+            body: value.body,
+        }
+    }
+}
+
 impl DocIndexer {
     pub fn new(embedding: Arc<dyn Embedding>) -> Self {
         let indexer = create_web_index(embedding);
         Self { indexer }
     }
 
-    pub async fn index_document(&self, document: SourceDocument) {
-        self.indexer.add(document).await;
+    pub async fn index_issue(&self, document: IssueDocument) {
+        self.indexer.add(document.into()).await;
+    }
+
+    pub fn commit(self) {
+        self.indexer.commit();
     }
 }

crates/tabby-scheduler/src/lib.rs

@@ -9,7 +9,7 @@ use async_stream::stream;
 pub use code::CodeIndexer;
 use crawl::crawl_pipeline;
 use doc::create_web_index;
-pub use doc::{DocIndexer, SourceDocument};
+pub use doc::{DocIndexer, IssueDocument};
 use futures::StreamExt;
 pub use indexer::{IndexAttributeBuilder, Indexer};
 
@@ -19,6 +19,8 @@ use std::{env, sync::Arc};
 use tokio_cron_scheduler::{Job, JobScheduler};
 use tracing::{debug, info, warn};
 
+use crate::doc::SourceDocument;
+
 pub async fn scheduler(now: bool, config: &tabby_common::config::Config) {
     if now {
         scheduler_pipeline(config).await;

ee/tabby-webserver/src/service/background_job/mod.rs

@@ -82,7 +82,7 @@ pub async fn start(
                         warn!("Sync integration job failed: {:?}", err);
                     }
 
-                    if let Err(err) = IndexIssuesJob::cron(now, sender.clone(), integration_service.clone()).await {
+                    if let Err(err) = IndexIssuesJob::cron(now, sender.clone(), third_party_repository_service.clone()).await {
                         warn!("Index issues job failed: {err:?}");
                     }
                 },

ee/tabby-webserver/src/service/background_job/third_party_integration.rs

@@ -63,16 +63,16 @@ impl SyncIntegrationJob {
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct IndexIssuesJob {
-    integration_id: ID,
+    repository_id: ID,
 }
 
 impl Job for IndexIssuesJob {
     const NAME: &'static str = "index_issues";
 }
 
 impl IndexIssuesJob {
-    pub fn new(integration_id: ID) -> Self {
-        Self { integration_id }
+    pub fn new(repository_id: ID) -> Self {
+        Self { repository_id }
     }
 
     pub async fn run(
@@ -81,50 +81,42 @@ impl IndexIssuesJob {
         repository_service: Arc<dyn ThirdPartyRepositoryService>,
         integration_service: Arc<dyn IntegrationService>,
     ) -> tabby_schema::Result<()> {
+        let repository = repository_service
+            .get_provided_repository(self.repository_id)
+            .await?;
         let integration = integration_service
-            .get_integration(self.integration_id)
+            .get_integration(repository.integration_id.clone())
             .await?;
 
-        for repository in repository_service
-            .list_repositories_with_filter(
-                Some(vec![integration.id.clone()]),
-                None,
-                Some(true),
-                None,
-                None,
-                None,
-                None,
-            )
-            .await?
-        {
-            match &integration.kind {
-                IntegrationKind::Github | IntegrationKind::GithubSelfHosted => {
-                    index_github_issues(
-                        embedding.clone(),
-                        integration.api_base(),
-                        &repository.display_name,
-                        &integration.access_token,
-                    )
-                    .await?;
-                }
-                IntegrationKind::Gitlab | IntegrationKind::GitlabSelfHosted => {}
+        debug!("Indexing issues for repository {}", repository.display_name);
+
+        match &integration.kind {
+            IntegrationKind::Github | IntegrationKind::GithubSelfHosted => {
+                index_github_issues(
+                    embedding.clone(),
+                    integration.api_base(),
+                    &repository.display_name,
+                    &integration.access_token,
+                )
+                .await?;
             }
+            IntegrationKind::Gitlab | IntegrationKind::GitlabSelfHosted => {}
         }
         Ok(())
     }
 
     pub async fn cron(
         _now: DateTime<Utc>,
         sender: tokio::sync::mpsc::UnboundedSender<BackgroundJobEvent>,
-        integration_service: Arc<dyn IntegrationService>,
+        repository_service: Arc<dyn ThirdPartyRepositoryService>,
     ) -> tabby_schema::Result<()> {
-        for integration in integration_service
-            .list_integrations(None, None, None, None, None, None)
+        for repository in repository_service
+            .list_repositories_with_filter(None, None, Some(true), None, None, None, None)
             .await?
         {
             sender
-                .send(BackgroundJobEvent::IndexIssues(integration.id))
-                .map_err(|_| anyhow!("Failed to enqueue issue index job"))?;
+                .send(BackgroundJobEvent::IndexIssues(repository.id))
+                .map_err(|_| anyhow!("Failed to enqueue scheduler job"))?;
         }
         Ok(())
     }

ee/tabby-webserver/src/service/background_job/third_party_integration/issues.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use anyhow::{anyhow, Result};
 use octocrab::{models::issues::Issue, Octocrab};
 use tabby_inference::Embedding;
-use tabby_scheduler::{DocIndexer, SourceDocument};
+use tabby_scheduler::{DocIndexer, IssueDocument};
 
 async fn fetch_github_issues(
     api_base: &str,
@@ -26,6 +26,7 @@ async fn fetch_github_issues(
         let response = octocrab
             .issues(owner, repo)
             .list()
+            .state(octocrab::params::State::All)
             .page(page)
             .send()
             .await?;
@@ -51,13 +52,15 @@ pub async fn index_github_issues(
     let index = DocIndexer::new(embedding);
 
     for issue in fetch_github_issues(api_base, full_name, access_token).await? {
-        let document = SourceDocument {
+        let document = IssueDocument {
             id: format!("{full_name}/issues/{}", issue.id),
             title: issue.title,
             link: issue.url.to_string(),
             body: issue.body.unwrap_or_default(),
         };
-        index.index_document(document).await;
+        index.index_issue(document).await;
     }
+
+    index.commit();
     Ok(())
 }

ee/tabby-webserver/src/service/repository/third_party.rs

@@ -134,6 +134,10 @@ impl ThirdPartyRepositoryService for ThirdPartyRepositoryServiceImpl {
                 .send(BackgroundJobEvent::Scheduler(RepositoryConfig::new(
                     git_url,
                 )));
+
+            let _ = self
+                .background_job
+                .send(BackgroundJobEvent::IndexIssues(repo.id));
         }
 
         Ok(())