From cb50479ab2483c588078d667f75c0e3afd5b1a70 Mon Sep 17 00:00:00 2001 From: ManiMatter <124743318+ManiMatter@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:08:04 +0200 Subject: [PATCH 1/4] Update get-versions.ts --- src/version/get-versions.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index 342cad2b..e05c5753 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -70,7 +70,16 @@ export function getOldestVersions( id: version.id, version: version.name, created_at: version.created_at, - tagged + tagged, + subIds: [] <<<<<<<<<<<<<<<<<<<<<<<<<<<<< First, we need to get for all packages the subIds (if they have any) + Example: https://github.com/ManiMatter/decluttarr/pkgs/container/decluttarr/204442395?tag=v1.38.0 + Package v1.38.0: sha256:b4a9b04d8c0a5ab9f400f7f64f8be20d9951a996fd00882a936087af8f5ce43d + Has 3 Sub-IDs: + linux/amd64: sha256:c2dfb515fd9a6ad396fe6a48cd3e535b4079b467cb691bcb3faede6889089d6e + linux/arm64: sha256:59b2aa2e04cc6b3391f612833e87bbd0c4fdfddb04845b8e8f0365a45e90151c + unknown/unknown: sha256:6dfc07ab69cbe95303f51fed14b40a9574bbebbb3501d7aec481d184a8321c91 + + } }), page, From 6aa53feb10fa01d131216f8ceee0c2cd809cf66e Mon Sep 17 00:00:00 2001 From: ManiMatter <124743318+ManiMatter@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:22:05 +0200 Subject: [PATCH 2/4] Update delete.ts --- src/delete.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/delete.ts b/src/delete.ts index 4eb953fe..0dc84748 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -74,10 +74,31 @@ export function finalIds(input: Input): Observable { Here first filter out the versions that are to be ignored. Then compute number of versions to delete (toDelete) based on the inputs. */ + allValues = value ------------- Make a copy of all values value = value.filter(info => !input.ignoreVersions.test(info.version)) if (input.deleteUntaggedVersions === 'true') { - value = value.filter(info => !info.tagged) + // Previous Code: + // This loses untagged packages that belong to multi-arch containers that are tagged + // value = value.filter(info => !info.tagged) + + // PSEUDOCODE TO FIX THIS: + // Step 1: Save a copy of the original value array into valueAll + let valueAll = value + + // Step 2: Save list of IDs that are tagged + const taggedIDs = valueAll.filter(info => info.tagged).map(info => info.id); + + // Step 3: Create a list of IDs that not tagged, but they are subIDs and the parent ID (multi-arch container) is tagged + const subIDs = valueAll + .filter(info => info.tagged) + .flatMap(info => info.subIDs); + + // Step 4: Filter valueAll based on the IDs obtained in steps 2 and 3 -> Only those should be removed + value = valueAll.filter(info => !taggedIDs.includes(info.id) && !subIDs.includes(info.id)); + + // Result: value is now filtered and does not contain any tagged packages, or untagged packages that relate to multi-arch packages where the parent package is not tagged + } let toDelete = 0 From 4425d6400c644f3ef2f87e47d1b86716df1baa80 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Wed, 17 Apr 2024 13:13:05 +0200 Subject: [PATCH 3/4] Exclude subpackages in multi-arch containers when establishing how many packages to retain (under minVersionsToKeep) --- src/delete.ts | 79 ++++++++++++++++++++++++++----------- src/version/get-versions.ts | 4 +- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/delete.ts b/src/delete.ts index 0dc84748..3fd72f61 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -70,35 +70,45 @@ export function finalIds(input: Input): Observable { new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ) }) + /* Here first filter out the versions that are to be ignored. Then compute number of versions to delete (toDelete) based on the inputs. - */ - allValues = value ------------- Make a copy of all values + */ value = value.filter(info => !input.ignoreVersions.test(info.version)) + /* + Step 0: + Iterate over value to extract subIDs and create a new array. + This new array just contains the subIDs and shows to which parentID (= Multi Arch package) the are linked and whether that parentId is tagged + */ + value.forEach(value => { + const { id, tagged, subIds } = version; + + // Add parentID to the subIDs array with tagged status + subIds.forEach(subId => { + subIdsArray.push({ + subId: subId, + parentId: id, + tagged: tagged + }); + }); + }); + + + if (input.deleteUntaggedVersions === 'true') { // Previous Code: - // This loses untagged packages that belong to multi-arch containers that are tagged // value = value.filter(info => !info.tagged) + // Problem: This loses untagged packages that belong to multi-arch containers that are tagged + + // PSEUDOCODE TO FIX THIS: + // Keep only packages that are not subpackages and not tagged; or that are subpackages and the parent package is not tagged + value = value.filter(info => + (!info.tagged && !subIdsArray.some(subIdInfo => subIdInfo.subId === info.id)) || + (subIdsArray.some(subIdInfo => subIdInfo.subId === info.id && !subIdInfo.tagged)) + ); - // PSEUDOCODE TO FIX THIS: - // Step 1: Save a copy of the original value array into valueAll - let valueAll = value - - // Step 2: Save list of IDs that are tagged - const taggedIDs = valueAll.filter(info => info.tagged).map(info => info.id); - - // Step 3: Create a list of IDs that not tagged, but they are subIDs and the parent ID (multi-arch container) is tagged - const subIDs = valueAll - .filter(info => info.tagged) - .flatMap(info => info.subIDs); - - // Step 4: Filter valueAll based on the IDs obtained in steps 2 and 3 -> Only those should be removed - value = valueAll.filter(info => !taggedIDs.includes(info.id) && !subIDs.includes(info.id)); - - // Result: value is now filtered and does not contain any tagged packages, or untagged packages that relate to multi-arch packages where the parent package is not tagged - } let toDelete = 0 @@ -108,13 +118,38 @@ export function finalIds(input: Input): Observable { Math.min(input.numOldVersionsToDelete, RATE_LIMIT) ) } else { + /* + Keeps only n packages + If deleteUntaggedVersions is true, then all tagged versions are kept, plus a number of untagged versions (minVersionsToKeep) + If deleteUntaggedVersions is false, then just n versions should remain. + In both cases, sub-packages should not be counted (keen n real packages, not n subpackages) + Problem with current code: when determining n-versions, also subpackages are counted (rather than just counting real packages) + */ + // PSEUDOCODE TO FIX THIS: + // Step 1: Create an array that does not include the subpackages to calculate which ones need to be retained + const valueWithoutSub = value.filter(info => (!subIdsArray.some(subIdInfo => subIdInfo.subId === info.id))) toDelete = Math.min( - value.length - input.minVersionsToKeep, + valueWithoutSub.length - input.minVersionsToKeep, RATE_LIMIT ) } if (toDelete < 0) return [] - return value.map(info => info.id.toString()).slice(0, toDelete) + + // Step 2: Filter out the parent packages that need to be retained + valueWithoutSub = valueWithoutSub.map(info => info.id.toString()).slice(0, toDelete) + + // Step 3: Filter out from value all packages that meet either of these two conditions: + // a. all parent packages flagged for deletion + // b. all subpackages that belong to these parent packages that are flagged for deletion + value = value.filter(info => + valueWithoutSub.some(item => item.id === info.id) || + subIdsArray.some(subIdInfo => + subIdInfo.subId === info.id && + valueWithoutSub.some(item => item.id === subIdInfo.parentId) + ) + ); + + return value }) ) } diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index e05c5753..172b5f8c 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -71,14 +71,14 @@ export function getOldestVersions( version: version.name, created_at: version.created_at, tagged, - subIds: [] <<<<<<<<<<<<<<<<<<<<<<<<<<<<< First, we need to get for all packages the subIds (if they have any) + subIds: [] /* <<<<<<<<<<<<<<<<<<<<<<<<<<<<< First, we need to get for all packages the subIds (if they have any) Example: https://github.com/ManiMatter/decluttarr/pkgs/container/decluttarr/204442395?tag=v1.38.0 Package v1.38.0: sha256:b4a9b04d8c0a5ab9f400f7f64f8be20d9951a996fd00882a936087af8f5ce43d Has 3 Sub-IDs: linux/amd64: sha256:c2dfb515fd9a6ad396fe6a48cd3e535b4079b467cb691bcb3faede6889089d6e linux/arm64: sha256:59b2aa2e04cc6b3391f612833e87bbd0c4fdfddb04845b8e8f0365a45e90151c unknown/unknown: sha256:6dfc07ab69cbe95303f51fed14b40a9574bbebbb3501d7aec481d184a8321c91 - + */ } }), From 333aeb21f72d19a55c238da7d501a907cab7217e Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Wed, 17 Apr 2024 13:16:41 +0200 Subject: [PATCH 4/4] Exclude subpackages in multi-arch containers when establishing how many packages to retain (under minVersionsToKeep) --- src/delete.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/delete.ts b/src/delete.ts index 3fd72f61..4131549c 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -112,12 +112,6 @@ export function finalIds(input: Input): Observable { } let toDelete = 0 - if (input.minVersionsToKeep < 0) { - toDelete = Math.min( - value.length, - Math.min(input.numOldVersionsToDelete, RATE_LIMIT) - ) - } else { /* Keeps only n packages If deleteUntaggedVersions is true, then all tagged versions are kept, plus a number of untagged versions (minVersionsToKeep) @@ -126,8 +120,15 @@ export function finalIds(input: Input): Observable { Problem with current code: when determining n-versions, also subpackages are counted (rather than just counting real packages) */ // PSEUDOCODE TO FIX THIS: - // Step 1: Create an array that does not include the subpackages to calculate which ones need to be retained - const valueWithoutSub = value.filter(info => (!subIdsArray.some(subIdInfo => subIdInfo.subId === info.id))) + // Step 1: Create an array that does not include the subpackages to calculate which ones need to be retained / deleted + const valueWithoutSub = value.filter(info => (!subIdsArray.some(subIdInfo => subIdInfo.subId === info.id))) + + if (input.minVersionsToKeep < 0) { + toDelete = Math.min( + valueWithoutSub.length, + Math.min(input.numOldVersionsToDelete, RATE_LIMIT) + ) + } else { toDelete = Math.min( valueWithoutSub.length - input.minVersionsToKeep, RATE_LIMIT