[backend] Add more logs to understand progression of migration

OpenCTI-Platform · Feb 1, 2025 · 87ac343 · 87ac343
1 parent 8d8ee39
commit 87ac343
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js
@@ -3383,8 +3383,9 @@ export const elPaginate = async (context, user, indexName, options = {}) => {
     );
 };
 export const elList = async (context, user, indexName, opts = {}) => {
-  const { maxSize = undefined } = opts;
+  const { maxSize = undefined, logForMigration = false } = opts;
   const first = opts.first ?? ES_DEFAULT_PAGINATION;
+  let batch = 0;
   let emitSize = 0;
   let hasNextPage = true;
   let continueProcess = true;
@@ -3407,6 +3408,10 @@ export const elList = async (context, user, indexName, opts = {}) => {
     const noMoreElements = elements.length === 0 || elements.length < first;
     const moreThanMax = maxSize ? emitSize >= maxSize : false;
     if (noMoreElements || moreThanMax) {
+      batch += 1;
+      if( logForMigration ) {
+        logMigration.info(`Migrating batch ${batch}...`)
+      }
       if (elements.length > 0) {
         await publish(elements);
       }

diff --git a/opencti-platform/opencti-graphql/src/migrations/1733912620469-clean-deprecated-rels.js b/opencti-platform/opencti-graphql/src/migrations/1733912620469-clean-deprecated-rels.js
@@ -68,7 +68,7 @@ export const up = async (next) => {
     ENTITY_TYPE_MALWARE,
     ENTITY_TYPE_INCIDENT,
   ];
-  const optsRelatedTo = { types: threatTypes, callback: clearRelatedToObservableRels };
+  const optsRelatedTo = { types: threatTypes, logForMigration: true, callback: clearRelatedToObservableRels };
   await elList(context, SYSTEM_USER, READ_INDEX_STIX_DOMAIN_OBJECTS, optsRelatedTo);
   // Apply operations.
   let currentProcessingRelatedTo = 0;
@@ -118,7 +118,7 @@ export const up = async (next) => {
       currentProcessingLocations += 1;
     }
   };
-  const opts = { types: [ENTITY_TYPE_LOCATION_REGION, ENTITY_TYPE_LOCATION_COUNTRY], callback: clearLocatedAtRegionAndCountryRels };
+  const opts = { types: [ENTITY_TYPE_LOCATION_REGION, ENTITY_TYPE_LOCATION_COUNTRY], logForMigration: true, callback: clearLocatedAtRegionAndCountryRels };
   await elList(context, SYSTEM_USER, READ_INDEX_STIX_DOMAIN_OBJECTS, opts);
   // Apply operations.
   let currentProcessing = 0;