Use bulk delete objects instea of single calls

trustfractal · Jul 14, 2022 · 42a81b5 · pkoch · Jul 14, 2022 · 42a81b5
1 parent 34c8f9d
commit 42a81b5
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 6 deletions.
diff --git a/dist/index.js b/dist/index.js
@@ -40702,6 +40702,7 @@ const AWS = __nccwpck_require__(71786);
 const wmatch = __nccwpck_require__(77309);
 
 const s3 = new AWS.S3();
+const MAX_OBJECT_COUNT = 500;
 
 const cleanArray = (value) => (value || [])
   .map(x => x.trim())
@@ -40728,6 +40729,7 @@ const main = async () => {
   core.debug("Iterate over files and test include, exclude filters and dates...");
   let isTruncated = true;
   let lastMarker = undefined;
+  let objects = [];
 
   try {
     while (isTruncated) {
@@ -40758,10 +40760,21 @@ const main = async () => {
         if (dryRun) {
           console.log(`-> Item ${item.Key} will be removed`);
         } else {
-          await s3.deleteObject({
-            Bucket: bucket,
+          objects.push({
             Key: item.Key,
-          }).promise();
+            VersionId: item.VersionId,
+          });
+
+          if (objects.length > MAX_OBJECT_COUNT) {
+            await s3.deleteObjects({
+              Bucket: bucket,
+              Delete: {
+                Objects: objects,
+              },
+            }).promise();
+
+            objects = [];
+          }
         }
       }
 
@@ -40771,6 +40784,15 @@ const main = async () => {
         lastMarker = rest.NextMarker;
       }
     }
+
+    if (objects.length) {
+      await s3.deleteObjects({
+        Bucket: bucket,
+        Delete: {
+          Objects: objects,
+        },
+      }).promise();
+    }
   } catch (e) {
     core.setFailed(e);
   }

diff --git a/index.js b/index.js
@@ -3,6 +3,7 @@ const AWS = require("aws-sdk");
 const wmatch = require("wildcard-match");
 
 const s3 = new AWS.S3();
+const MAX_OBJECT_COUNT = 500;
 
 const cleanArray = (value) => (value || [])
   .map(x => x.trim())
@@ -29,6 +30,7 @@ const main = async () => {
   core.debug("Iterate over files and test include, exclude filters and dates...");
   let isTruncated = true;
   let lastMarker = undefined;
+  let objects = [];
 
   try {
     while (isTruncated) {
@@ -59,10 +61,21 @@ const main = async () => {
         if (dryRun) {
           console.log(`-> Item ${item.Key} will be removed`);
         } else {
-          await s3.deleteObject({
-            Bucket: bucket,
+          objects.push({
             Key: item.Key,
-          }).promise();
+            VersionId: item.VersionId,
+          });
+
+          if (objects.length > MAX_OBJECT_COUNT) {
+            await s3.deleteObjects({
+              Bucket: bucket,
+              Delete: {
+                Objects: objects,
+              },
+            }).promise();
+
+            objects = [];
+          }
         }
       }
 
@@ -72,6 +85,15 @@ const main = async () => {
         lastMarker = rest.NextMarker;
       }
     }
+
+    if (objects.length) {
+      await s3.deleteObjects({
+        Bucket: bucket,
+        Delete: {
+          Objects: objects,
+        },
+      }).promise();
+    }
   } catch (e) {
     core.setFailed(e);
   }