From 9a4f79bde47dfa374119f74eba25723ab0e8bf5b Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 31 Jan 2024 11:41:25 -0500 Subject: [PATCH 01/10] create CacheExpiryCheck for automated removal of expired cached data --- .../distrib/cachemgr/CacheExpiryCheck.java | 62 +++++++++++ .../cachemgr/CacheExpiryCheckTest.java | 102 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java create mode 100644 src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java new file mode 100644 index 00000000..64a1bd56 --- /dev/null +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -0,0 +1,62 @@ +package gov.nist.oar.distrib.cachemgr; + +import gov.nist.oar.distrib.StorageVolumeException; + +/** + * Implements a cache object check to identify and remove objects that have been in the cache + * longer than a specified duration, specifically two weeks. This check helps in + * managing cache integrity by ensuring that stale or outdated data are removed + * from the cache. + */ +public class CacheExpiryCheck implements CacheObjectCheck { + + private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds + private StorageInventoryDB inventoryDB; + + public CacheExpiryCheck(StorageInventoryDB inventoryDB) { + this.inventoryDB = inventoryDB; + } + + /** + * Checks whether a cache object has expired based on its last modified time and removes it if expired. + * An object is considered expired if it has been in the cache for more than two weeks. + * + * @param co The CacheObject to be checked for expiry. + * @throws IntegrityException if the cache object's last modified time is unknown. + * @throws StorageVolumeException if there is an issue removing the expired object from the cache volume. + */ + @Override + public void check(CacheObject co) throws IntegrityException, StorageVolumeException { + long currentTime = System.currentTimeMillis(); + long objectLastModifiedTime = co.getLastModified(); + + // Throw an exception if the last modified time is unknown + if (objectLastModifiedTime == -1) { + throw new IntegrityException("Last modified time of cache object is unknown: " + co.name); + } + + // If the cache object is expired, remove it from the cache + if ((currentTime - objectLastModifiedTime) > TWO_WEEKS_MILLIS) { + removeExpiredObject(co); + } + } + + /** + * Removes an expired object from the cache. + * + * @param co The expired CacheObject to be removed. + * @throws StorageVolumeException if there is an issue removing the object from the cache volume. + */ + protected void removeExpiredObject(CacheObject co) throws StorageVolumeException { + CacheVolume volume = co.volume; + if (volume != null && volume.remove(co.name)) { + try { + inventoryDB.removeObject(co.volname, co.name); + } catch (InventoryException e) { + throw new StorageVolumeException("Failed to remove object from inventory database: " + co.name, e); + } + } else { + throw new StorageVolumeException("Failed to remove expired object from cache volume: " + co.name); + } + } +} diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java new file mode 100644 index 00000000..dad89ab7 --- /dev/null +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -0,0 +1,102 @@ +package gov.nist.oar.distrib.cachemgr; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class CacheExpiryCheckTest { + + @Mock + private StorageInventoryDB mockInventoryDB; + @Mock + private CacheVolume mockVolume; + @Mock + private CacheObject cacheObject; + private CacheExpiryCheck expiryCheck; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + expiryCheck = new CacheExpiryCheck(mockInventoryDB); + } + + /** + * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. + * An object is considered expired if its last modified time is more than two weeks ago. + * This test checks that the expired object is appropriately removed from both the cache volume and the inventory + * database. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testExpiredObject() throws Exception { + // Setup mock + cacheObject.name = "testObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000); // 15 days in milliseconds + when(cacheObject.getLastModified()).thenReturn(lastModified); + + + when(mockVolume.remove("testObject")).thenReturn(true); + + // Perform the check + expiryCheck.check(cacheObject); + + // Verify the interactions + verify(mockVolume).remove("testObject"); + verify(mockInventoryDB).removeObject(anyString(), anyString()); + } + + + /** + * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if it has been + * modified within the last two weeks. This test checks that no removal action is taken for a non-expired object. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testNonExpiredObject() throws Exception { + // Setup mock + cacheObject.name = "nonExpiredObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + + long lastModified = System.currentTimeMillis() - (5 * 24 * 60 * 60 * 1000); // 5 days in milliseconds + when(cacheObject.getLastModified()).thenReturn(lastModified); + + // Perform the check + expiryCheck.check(cacheObject); + + // Verify that the remove method was not called as the object is not expired + verify(mockVolume, never()).remove("nonExpiredObject"); + verify(mockInventoryDB, never()).removeObject("testVolume", "nonExpiredObject"); + } + + /** + * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when the last modified time + * of a cache object is unknown (indicated by a value of -1). This situation should be flagged as an error + * as the expiry status of the object cannot be determined. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test(expected = IntegrityException.class) + public void testUnknownLastModifiedTime() throws Exception { + // Setup mock + cacheObject.name = "unknownLastModifiedObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + long lastModified = -1; // Unknown last modified time + when(cacheObject.getLastModified()).thenReturn(lastModified); + + // Perform the check, expecting an IntegrityException + expiryCheck.check(cacheObject); + } + +} From ae225b20224a161ca78e29f89f74c516af7dd8e6 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 31 Jan 2024 17:04:24 -0500 Subject: [PATCH 02/10] add the CacheExpiryCheck to the list of checks --- .../gov/nist/oar/distrib/web/NISTCacheManagerConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java index cfb74fab..da4d674d 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java @@ -12,9 +12,11 @@ package gov.nist.oar.distrib.web; import gov.nist.oar.distrib.cachemgr.BasicCache; +import gov.nist.oar.distrib.cachemgr.CacheExpiryCheck; import gov.nist.oar.distrib.cachemgr.ConfigurableCache; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.CacheVolume; +import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.storage.AWSS3CacheVolume; import gov.nist.oar.distrib.cachemgr.storage.FilesystemCacheVolume; import gov.nist.oar.distrib.cachemgr.VolumeStatus; @@ -485,6 +487,9 @@ public PDRCacheManager createCacheManager(BasicCache cache, PDRDatasetRestorer r List checks = new ArrayList(); checks.add(new ChecksumCheck(false, true)); + // Get the StorageInventoryDB from the cache and add the CacheExpiryCheck to the list of checks + StorageInventoryDB inventoryDB = cache.getInventoryDB(); + checks.add(new CacheExpiryCheck(inventoryDB)); PDRCacheManager out = new PDRCacheManager(cache, rstr, checks, getCheckDutyCycle()*1000, getCheckGracePeriod()*1000, -1, rootdir, logger); if (getMonitorAutoStart()) { From c73ce93baf8a4f852139e682204514787cab8b81 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 08:43:03 -0500 Subject: [PATCH 03/10] Implement dynamic cache expiration based on expiresIn metadata --- .../distrib/cachemgr/CacheExpiryCheck.java | 58 ++++++++++--------- .../cachemgr/CacheExpiryCheckTest.java | 49 ++++++++++------ 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index 64a1bd56..5a82d2bd 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -2,6 +2,8 @@ import gov.nist.oar.distrib.StorageVolumeException; +import java.time.Instant; + /** * Implements a cache object check to identify and remove objects that have been in the cache * longer than a specified duration, specifically two weeks. This check helps in @@ -10,7 +12,7 @@ */ public class CacheExpiryCheck implements CacheObjectCheck { - private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds + // private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds private StorageInventoryDB inventoryDB; public CacheExpiryCheck(StorageInventoryDB inventoryDB) { @@ -18,45 +20,45 @@ public CacheExpiryCheck(StorageInventoryDB inventoryDB) { } /** - * Checks whether a cache object has expired based on its last modified time and removes it if expired. - * An object is considered expired if it has been in the cache for more than two weeks. + * Checks if a cache object is expired and removes it from the cache if it is. + * The method uses the {@code expiresIn} metadata field to determine the expiration status. + * The expiration time is calculated based on the {@code LastModified} time plus the {@code expiresIn} duration. + * If the current time is past the calculated expiry time, the object is removed from the inventory database. * - * @param co The CacheObject to be checked for expiry. - * @throws IntegrityException if the cache object's last modified time is unknown. - * @throws StorageVolumeException if there is an issue removing the expired object from the cache volume. + * @param co The cache object to check for expiration. + * @throws IntegrityException If the object is found to be corrupted during the check. + * @throws StorageVolumeException If there's an error accessing the storage volume during the check. + * @throws CacheManagementException If there's an error managing the cache, including removing the expired object. */ @Override - public void check(CacheObject co) throws IntegrityException, StorageVolumeException { - long currentTime = System.currentTimeMillis(); - long objectLastModifiedTime = co.getLastModified(); + public void check(CacheObject co) throws IntegrityException, StorageVolumeException, CacheManagementException { + if (co == null || inventoryDB == null) { + throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); + } - // Throw an exception if the last modified time is unknown - if (objectLastModifiedTime == -1) { - throw new IntegrityException("Last modified time of cache object is unknown: " + co.name); + if (!co.hasMetadatum("expiresIn")) { + throw new IntegrityException("CacheObject missing 'expiresIn' metadata"); } - // If the cache object is expired, remove it from the cache - if ((currentTime - objectLastModifiedTime) > TWO_WEEKS_MILLIS) { - removeExpiredObject(co); + long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); + if (expiresInDuration == -1L) { + throw new IntegrityException("Invalid 'expiresIn' metadata value"); } - } - /** - * Removes an expired object from the cache. - * - * @param co The expired CacheObject to be removed. - * @throws StorageVolumeException if there is an issue removing the object from the cache volume. - */ - protected void removeExpiredObject(CacheObject co) throws StorageVolumeException { - CacheVolume volume = co.volume; - if (volume != null && volume.remove(co.name)) { + long lastModified = co.getLastModified(); + if (lastModified == -1L) { + throw new IntegrityException("CacheObject 'lastModified' time not available"); + } + + long expiryTime = lastModified + expiresInDuration; + long currentTime = Instant.now().toEpochMilli(); + + if (expiryTime < currentTime) { try { inventoryDB.removeObject(co.volname, co.name); } catch (InventoryException e) { - throw new StorageVolumeException("Failed to remove object from inventory database: " + co.name, e); + throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); } - } else { - throw new StorageVolumeException("Failed to remove expired object from cache volume: " + co.name); } } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index dad89ab7..0498092d 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -28,9 +28,9 @@ public void setUp() { /** * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. - * An object is considered expired if its last modified time is more than two weeks ago. - * This test checks that the expired object is appropriately removed from both the cache volume and the inventory - * database. + * An object is considered expired based on the `expiresIn` metadata, which defines the duration after which + * an object should be considered expired from the time of its last modification. This test ensures that an object + * past its expiration is appropriately removed from the inventory database. * * @throws Exception to handle any exceptions thrown during the test execution */ @@ -39,25 +39,23 @@ public void testExpiredObject() throws Exception { // Setup mock cacheObject.name = "testObject"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000); // 15 days in milliseconds + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000L); // 15 days ago when(cacheObject.getLastModified()).thenReturn(lastModified); - - when(mockVolume.remove("testObject")).thenReturn(true); - // Perform the check expiryCheck.check(cacheObject); // Verify the interactions - verify(mockVolume).remove("testObject"); - verify(mockInventoryDB).removeObject(anyString(), anyString()); + verify(mockInventoryDB).removeObject(cacheObject.volname, cacheObject.name); } /** - * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if it has been - * modified within the last two weeks. This test checks that no removal action is taken for a non-expired object. + * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if the current time has not + * exceeded its `expiresIn` duration since its last modification. This test verifies that no removal action is taken + * for such non-expired objects. * * @throws Exception to handle any exceptions thrown during the test execution */ @@ -66,17 +64,16 @@ public void testNonExpiredObject() throws Exception { // Setup mock cacheObject.name = "nonExpiredObject"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - - long lastModified = System.currentTimeMillis() - (5 * 24 * 60 * 60 * 1000); // 5 days in milliseconds + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + long lastModified = System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000L); // 7 days ago, within expiry period when(cacheObject.getLastModified()).thenReturn(lastModified); // Perform the check expiryCheck.check(cacheObject); // Verify that the remove method was not called as the object is not expired - verify(mockVolume, never()).remove("nonExpiredObject"); - verify(mockInventoryDB, never()).removeObject("testVolume", "nonExpiredObject"); + verify(mockInventoryDB, never()).removeObject(cacheObject.volname, cacheObject.name); } /** @@ -99,4 +96,22 @@ public void testUnknownLastModifiedTime() throws Exception { expiryCheck.check(cacheObject); } + /** + * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when a cache object lacks the + * `expiresIn` metadata. This scenario indicates that the object's expiration cannot be determined, necessitating + * error handling. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test(expected = IntegrityException.class) + public void testObjectWithoutExpiresInMetadata() throws Exception { + // Setup mock to simulate an object without expiresIn metadata + cacheObject.name = "objectWithoutExpiresIn"; + cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + + // Attempt to check the object, expecting an IntegrityException + expiryCheck.check(cacheObject); + } + } From 53a82a0b6230ae288cb7cee3d2ac7e99edca2dfa Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 08:50:25 -0500 Subject: [PATCH 04/10] Reorder checks - checksum check last --- .../java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java index da4d674d..4684f5ae 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java @@ -486,10 +486,11 @@ public PDRCacheManager createCacheManager(BasicCache cache, PDRDatasetRestorer r throw new ConfigurationException(rootdir+": Not an existing directory"); List checks = new ArrayList(); - checks.add(new ChecksumCheck(false, true)); // Get the StorageInventoryDB from the cache and add the CacheExpiryCheck to the list of checks StorageInventoryDB inventoryDB = cache.getInventoryDB(); checks.add(new CacheExpiryCheck(inventoryDB)); + checks.add(new ChecksumCheck(false, true)); + PDRCacheManager out = new PDRCacheManager(cache, rstr, checks, getCheckDutyCycle()*1000, getCheckGracePeriod()*1000, -1, rootdir, logger); if (getMonitorAutoStart()) { From f4a279b570eec33b02750ddd11faf4fa82817ab9 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 11:17:32 -0500 Subject: [PATCH 05/10] update removal logic --- .../distrib/cachemgr/CacheExpiryCheck.java | 57 ++++++++---- .../cachemgr/CacheExpiryCheckTest.java | 88 +++++++++++++------ 2 files changed, 99 insertions(+), 46 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index 5a82d2bd..c5ef1845 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -36,29 +36,48 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); } - if (!co.hasMetadatum("expiresIn")) { - throw new IntegrityException("CacheObject missing 'expiresIn' metadata"); - } - - long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); - if (expiresInDuration == -1L) { - throw new IntegrityException("Invalid 'expiresIn' metadata value"); - } + if (co.hasMetadatum("expiresIn")) { + long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); + if (expiresInDuration == -1L) { + throw new IntegrityException("Invalid 'expiresIn' metadata value"); + } - long lastModified = co.getLastModified(); - if (lastModified == -1L) { - throw new IntegrityException("CacheObject 'lastModified' time not available"); - } + long lastModified = co.getLastModified(); + if (lastModified == -1L) { + throw new IntegrityException("CacheObject 'lastModified' time not available"); + } - long expiryTime = lastModified + expiresInDuration; - long currentTime = Instant.now().toEpochMilli(); + long expiryTime = lastModified + expiresInDuration; + long currentTime = Instant.now().toEpochMilli(); - if (expiryTime < currentTime) { - try { - inventoryDB.removeObject(co.volname, co.name); - } catch (InventoryException e) { - throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); + // Check if the object is expired + if (expiryTime < currentTime) { + try { + boolean removed = removeObject(co); + if (!removed) { + throw new CacheManagementException("Failed to remove expired object: " + co.name); + } + } catch (InventoryException e) { + throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); + } } } } + + /** + * Attempts to remove a cache object from both its physical volume and the inventory database. + * Synchronization ensures thread-safe removal operations. + * + * @param co The cache object to be removed. + * @return true if the object was successfully removed from its volume, false otherwise. + * @throws StorageVolumeException if an error occurs accessing the storage volume. + * @throws InventoryException if an error occurs updating the inventory database. + */ + protected boolean removeObject(CacheObject co) throws StorageVolumeException, InventoryException { + synchronized (inventoryDB) { + boolean out = co.volume.remove(co.name); + inventoryDB.removeObject(co.volname, co.name); + return out; + } + } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index 0498092d..6796b053 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -5,6 +5,8 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import java.time.Instant; + import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -35,19 +37,17 @@ public void setUp() { * @throws Exception to handle any exceptions thrown during the test execution */ @Test - public void testExpiredObject() throws Exception { - // Setup mock - cacheObject.name = "testObject"; - cacheObject.volname = "testVolume"; + public void testExpiredObjectRemoval() throws Exception { + // Setup an expired cache object + cacheObject.volume = mockVolume; when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds - long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000L); // 15 days ago - when(cacheObject.getLastModified()).thenReturn(lastModified); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(Instant.now().minusSeconds(10).toEpochMilli()); + when(cacheObject.volume.remove(cacheObject.name)).thenReturn(true); - // Perform the check expiryCheck.check(cacheObject); - // Verify the interactions + // Verify removeObject effect verify(mockInventoryDB).removeObject(cacheObject.volname, cacheObject.name); } @@ -77,41 +77,75 @@ public void testNonExpiredObject() throws Exception { } /** - * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when the last modified time - * of a cache object is unknown (indicated by a value of -1). This situation should be flagged as an error - * as the expiry status of the object cannot be determined. + * Tests that no action is taken and no exception is thrown for a cache object without the {@code expiresIn} metadata. + * This verifies that the absence of {@code expiresIn} metadata does not trigger any removal process or result in an error. * * @throws Exception to handle any exceptions thrown during the test execution */ - @Test(expected = IntegrityException.class) - public void testUnknownLastModifiedTime() throws Exception { - // Setup mock - cacheObject.name = "unknownLastModifiedObject"; + @Test + public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception { + cacheObject.name = "objectWithoutExpiresIn"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - long lastModified = -1; // Unknown last modified time - when(cacheObject.getLastModified()).thenReturn(lastModified); + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); - // Perform the check, expecting an IntegrityException expiryCheck.check(cacheObject); + + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); } /** - * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when a cache object lacks the - * `expiresIn` metadata. This scenario indicates that the object's expiration cannot be determined, necessitating - * error handling. + * Test to ensure that a cache object with an expiration date in the future is not removed from the cache. + * This test verifies the {@code check} method's correct behavior in handling non-expired objects based + * on the {@code expiresIn} metadata. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testNonExpiredObject_NoRemoval() throws Exception { + // Setup a non-expired cache object + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future + when(cacheObject.getLastModified()).thenReturn(System.currentTimeMillis()); + + expiryCheck.check(cacheObject); + + // Verify no removal happens + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); + } + + + /** + * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expiresIn} metadata + * but lacks a valid {@code lastModified} time. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test(expected = IntegrityException.class) - public void testObjectWithoutExpiresInMetadata() throws Exception { - // Setup mock to simulate an object without expiresIn metadata - cacheObject.name = "objectWithoutExpiresIn"; + public void testObjectWithExpiresInButNoLastModified_ThrowsException() throws Exception { + cacheObject.name = "objectWithNoLastModified"; cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(-1L); // Last modified not available + + expiryCheck.check(cacheObject); + } + + /** + * Test to verify that no action is taken for a cache object missing the {@code expiresIn} metadata. + * This test ensures that the absence of {@code expiresIn} metadata does not trigger any removal or error. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testObjectWithoutExpiresIn_NoAction() throws Exception { + // Setup an object without expiresIn metadata when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); - // Attempt to check the object, expecting an IntegrityException expiryCheck.check(cacheObject); + + // Verify no action is taken + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); } } From a71758c00b3d5613d77ae72c75788abfae55f911 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 21 Feb 2024 12:10:01 -0500 Subject: [PATCH 06/10] create rpa bagstore --- .../distrib/web/NISTDistribServiceConfig.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java index 2ae27f21..e7c3d261 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java @@ -179,6 +179,12 @@ public class NISTDistribServiceConfig { @Value("${distrib.packaging.allowedRedirects:1}") int allowedRedirects; + + @Value("${distrib.rpa.bagstore-location}") + private String rpaBagstore; + + @Value("${distrib.rpa.bagstore-mode}") + private String rpaMode; @Autowired AmazonS3 s3client; // set via getter below @Autowired BagStorage lts; // set via getter below @@ -205,6 +211,25 @@ else if (mode.equals("local")) } } + /** + * The storage service to use to access the bags, considering the mode and location. + */ + @Bean + public BagStorage getBagStorageService() throws ConfigurationException { + try { + if ("aws".equals(rpaMode) || "remote".equals(rpaMode)) { + return new AWSS3LongTermStorage(rpaBagstore, s3client); + } else if ("local".equals(rpaMode)) { + return new FilesystemLongTermStorage(rpaBagstore); + } else { + throw new ConfigurationException("distrib.rpa.bagstore-mode", "Unsupported storage mode: " + rpaMode); + } + } catch (FileNotFoundException ex) { + throw new ConfigurationException("distrib.rpa.bagstore-location", + "Storage Location not found: " + ex.getMessage(), ex); + } + } + /** * the client for access S3 storage */ From a0a77f1b6585b1547d3e55a3e9d0826cdbabf6ac Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 23 Feb 2024 06:27:49 -0500 Subject: [PATCH 07/10] add expiresIn metadatum while caching rpa data --- .../cachemgr/pdr/PDRDatasetRestorer.java | 9 ++++ .../pdr/RestrictedDatasetRestorerTest.java | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index 7aebf76d..fc17faa3 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -82,6 +82,8 @@ public class PDRDatasetRestorer implements Restorer, PDRConstants, PDRCacheRoles HeadBagCacheManager hbcm = null; long smszlim = 100000000L; // 100 MB Logger log = null; + private static final long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; // 2 weeks in milliseconds + /** * create the restorer @@ -650,6 +652,13 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C md.put("ediid", resmd.get("ediid")); md.put("cachePrefs", prefs); + // Add expiresIn metadata for files marked as ROLE_RESTRICTED_DATA + if ((prefs & ROLE_RESTRICTED_DATA) != 0) { + // Calculate the expiration time as current time + 2 weeks + long expiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + md.put("expiresIn", expiresIn); + } + // find space in the cache, and copy the data file into it try { resv = into.reserveSpace(ze.getSize(), prefs); diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java index dd11fadf..eff08566 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java @@ -406,5 +406,58 @@ public void testCacheFromBag() } + @Test + public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { + + assertTrue(! cache.isCached("mds1491/trial1.json")); + assertTrue(! cache.isCached("mds1491/trial2.json")); + assertTrue(! cache.isCached("mds1491/trial3/trial3a.json")); + assertTrue(! cache.isCached("mds1491/README.txt")); + + Set cached = rstr.cacheDataset("mds1491", null, cache, true, + PDRCacheRoles.ROLE_RESTRICTED_DATA, null); + assertTrue(cached.contains("trial1.json")); + assertTrue(cached.contains("trial2.json")); + assertTrue(cached.contains("README.txt")); + assertTrue(cached.contains("trial3/trial3a.json")); + assertEquals(4, cached.size()); + + assertTrue(cache.isCached("mds1491/trial1.json")); + assertTrue(cache.isCached("mds1491/trial2.json")); + assertTrue(cache.isCached("mds1491/README.txt")); + assertTrue(cache.isCached("mds1491/trial3/trial3a.json")); + + List found = cache.getInventoryDB().findObject("mds1491/trial1.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + // Verify the "expiresIn" value is approximately 2 weeks from the current time + long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; + long expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + long actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + // Check that the absolute difference between the expected and actual expiresIn values is less than 1000ms + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/trial2.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/README.txt", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + } + } From 411bd6b12a9cea80a1fe57b15c34378bddb30214 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 23 Feb 2024 06:31:51 -0500 Subject: [PATCH 08/10] cleanup: remove the rpa storage bean and unused imports --- .../distrib/web/NISTDistribServiceConfig.java | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java index e7c3d261..bdca6877 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java @@ -17,14 +17,8 @@ import gov.nist.oar.distrib.service.DefaultPreservationBagService; import gov.nist.oar.distrib.service.FileDownloadService; import gov.nist.oar.distrib.service.PreservationBagService; -import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.DefaultRPARequestHandlerService; -import gov.nist.oar.distrib.service.rpa.JKSKeyRetriever; -import gov.nist.oar.distrib.service.rpa.KeyRetriever; -import gov.nist.oar.distrib.service.rpa.RPARequestHandlerService; import gov.nist.oar.distrib.storage.AWSS3LongTermStorage; import gov.nist.oar.distrib.storage.FilesystemLongTermStorage; -import gov.nist.oar.distrib.cachemgr.CacheManagementException; import io.swagger.v3.oas.models.Components; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; @@ -37,7 +31,6 @@ import java.util.List; import java.io.BufferedReader; import java.io.FileNotFoundException; -import java.io.IOException; import javax.activation.MimetypesFileTypeMap; import org.springframework.boot.SpringApplication; @@ -46,7 +39,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.web.client.RestTemplate; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; @@ -179,12 +171,6 @@ public class NISTDistribServiceConfig { @Value("${distrib.packaging.allowedRedirects:1}") int allowedRedirects; - - @Value("${distrib.rpa.bagstore-location}") - private String rpaBagstore; - - @Value("${distrib.rpa.bagstore-mode}") - private String rpaMode; @Autowired AmazonS3 s3client; // set via getter below @Autowired BagStorage lts; // set via getter below @@ -211,24 +197,6 @@ else if (mode.equals("local")) } } - /** - * The storage service to use to access the bags, considering the mode and location. - */ - @Bean - public BagStorage getBagStorageService() throws ConfigurationException { - try { - if ("aws".equals(rpaMode) || "remote".equals(rpaMode)) { - return new AWSS3LongTermStorage(rpaBagstore, s3client); - } else if ("local".equals(rpaMode)) { - return new FilesystemLongTermStorage(rpaBagstore); - } else { - throw new ConfigurationException("distrib.rpa.bagstore-mode", "Unsupported storage mode: " + rpaMode); - } - } catch (FileNotFoundException ex) { - throw new ConfigurationException("distrib.rpa.bagstore-location", - "Storage Location not found: " + ex.getMessage(), ex); - } - } /** * the client for access S3 storage From 39fc9a349357ae8edde116b6ec1a02729130ce61 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 27 Feb 2024 11:01:40 -0500 Subject: [PATCH 09/10] make the new metadata field 'expiresIn' configurable --- .../cachemgr/pdr/PDRDatasetRestorer.java | 22 +++++--- .../pdr/RestrictedDatasetRestorer.java | 56 ++++++++++++------- .../web/RPACachingServiceProvider.java | 4 +- .../oar/distrib/web/RPAConfiguration.java | 11 +++- .../web/RPACachingServiceProviderTest.java | 4 +- .../oar/distrib/web/RPAConfigurationTest.java | 1 + src/test/resources/rpaconfig.json | 3 +- 7 files changed, 68 insertions(+), 33 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index fc17faa3..7dbfc390 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -82,8 +82,6 @@ public class PDRDatasetRestorer implements Restorer, PDRConstants, PDRCacheRoles HeadBagCacheManager hbcm = null; long smszlim = 100000000L; // 100 MB Logger log = null; - private static final long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; // 2 weeks in milliseconds - /** * create the restorer @@ -652,12 +650,8 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C md.put("ediid", resmd.get("ediid")); md.put("cachePrefs", prefs); - // Add expiresIn metadata for files marked as ROLE_RESTRICTED_DATA - if ((prefs & ROLE_RESTRICTED_DATA) != 0) { - // Calculate the expiration time as current time + 2 weeks - long expiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - md.put("expiresIn", expiresIn); - } + // a hook for handling the expiration logic + updateMetadata(md, prefs); // find space in the cache, and copy the data file into it try { @@ -696,6 +690,18 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C fixMissingChecksums(into, fix, manifest); } + /** + * Method intended for customization of metadata before caching. This method can be overridden + * by subclasses to implement specific metadata customization logic as needed. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + protected void updateMetadata(JSONObject md, int prefs) { + // Default implementation does nothing. + // Subclasses can override this to implement specific logic. + } + /** * helper method to generate an ID for the object to be cached */ diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java index da1255c0..d06dff33 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java @@ -20,44 +20,23 @@ import gov.nist.oar.distrib.ObjectNotFoundException; import gov.nist.oar.distrib.ResourceNotFoundException; import gov.nist.oar.distrib.BagStorage; -import gov.nist.oar.distrib.Checksum; -import gov.nist.oar.distrib.cachemgr.Restorer; import gov.nist.oar.distrib.cachemgr.Reservation; -import gov.nist.oar.distrib.cachemgr.IntegrityMonitor; -import gov.nist.oar.distrib.cachemgr.BasicCache; import gov.nist.oar.distrib.cachemgr.Cache; import gov.nist.oar.distrib.cachemgr.CacheObject; -import gov.nist.oar.distrib.cachemgr.CacheObjectCheck; -import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.RestorationException; -import gov.nist.oar.distrib.cachemgr.InventoryException; import java.util.Collection; -import java.util.List; -import java.util.ArrayList; import java.util.Set; -import java.util.Map; -import java.util.HashSet; -import java.util.HashMap; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipEntry; -import java.nio.file.Path; -import java.nio.file.Paths; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.BufferedReader; import java.io.IOException; import java.io.FileNotFoundException; -import java.text.ParseException; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.json.JSONObject; import org.json.JSONException; -import org.apache.commons.io.FilenameUtils; /** * A {@link gov.nist.oar.distrib.cachemgr.Restorer} for restoring "restricted public" datasets from the @@ -82,6 +61,7 @@ */ public class RestrictedDatasetRestorer extends PDRDatasetRestorer { BagStorage restrictedLtstore = null; + long expiryTime = 1209600000L; // 2 weeks in milliseconds /** * create the restorer @@ -125,6 +105,25 @@ public RestrictedDatasetRestorer(BagStorage publicLtstore, BagStorage restricted this.restrictedLtstore = restrictedLtstore; } + /** + * Retrieves the expiry time for data. + *

+ * This value represents the duration in milliseconds after which the data is considered expired. + * + * @return the expiry time in milliseconds. + */ + public long getExpiryTime() { + return expiryTime; + } + + /** + * Sets the expiry time for restricted/public access content. + * + * @param expiryTime the expiry time in milliseconds to set. + */ + public void setExpiryTime(long expiryTime) { + this.expiryTime = expiryTime; + } /** * return true if an object does not exist in the long term storage system. Returning @@ -285,4 +284,19 @@ protected void cacheFromBag(String bagfile, Collection need, Collection< target, ltstore); } } + + /** + * Updates the metadata for files marked as restricted, by adding an expiration time. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + @Override + protected void updateMetadata(JSONObject md, int prefs) { + if ((prefs & ROLE_RESTRICTED_DATA) != 0) { + // Calculate the expiration time as current time + expiryTime + long expiresIn = System.currentTimeMillis() + expiryTime; + md.put("expiresIn", expiresIn); + } + } } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java index 82f31b89..a59e81e3 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java @@ -177,7 +177,9 @@ else if (mode.equals("local")) public RestrictedDatasetRestorer createRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - return new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + RestrictedDatasetRestorer rdr = new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + rdr.setExpiryTime(rpacfg.getExpiresAfterMillis()); + return rdr; // return new PDRDatasetRestorer(getRPBagStorage(), getHeadBagCacheManager()); } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java index de4e226e..791fdbfc 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java @@ -53,7 +53,8 @@ public class RPAConfiguration { String bagStore = null; @JsonProperty("bagstore-mode") String mode = null; - + @JsonProperty("expiresAfterMillis") + long expiresAfterMillis = 0L; public long getHeadbagCacheSize() { return hbCacheSize; @@ -76,6 +77,14 @@ public void setBagstoreMode(String mode) { this.mode = mode; } + public long getExpiresAfterMillis() { + return expiresAfterMillis; + } + + public void setExpiresAfterMillis(long expiresAfterMillis) { + this.expiresAfterMillis = expiresAfterMillis; + } + public SalesforceJwt getSalesforceJwt() { return salesforceJwt; } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java index 90060b54..1c5d8ac7 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java @@ -1,5 +1,6 @@ package gov.nist.oar.distrib.web; +import gov.nist.oar.distrib.cachemgr.pdr.RestrictedDatasetRestorer; import gov.nist.oar.distrib.storage.FilesystemLongTermStorage; import gov.nist.oar.distrib.BagStorage; import gov.nist.oar.distrib.cachemgr.CacheManagementException; @@ -81,6 +82,7 @@ public void testGetHeadBagManager() public void testCreateRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - PDRDatasetRestorer rest = prov.createRPDatasetRestorer(); + RestrictedDatasetRestorer rest = prov.createRPDatasetRestorer(); + assertEquals(rest.getExpiryTime(), 1209600000L); } } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java index f600fb62..0a5ae317 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java @@ -22,5 +22,6 @@ public void testLoadConfig() throws IOException { assertEquals("local", config.getBagstoreMode()); assertNull(config.getBagstoreLocation()); assertEquals("1234567890 pdr.rpa.2023 1234567890", config.getJwtSecretKey()); + assertEquals(1209600000L, config.getExpiresAfterMillis()); } } diff --git a/src/test/resources/rpaconfig.json b/src/test/resources/rpaconfig.json index 89441de8..19f809b3 100644 --- a/src/test/resources/rpaconfig.json +++ b/src/test/resources/rpaconfig.json @@ -3,5 +3,6 @@ "datacartUrl": "https://localhost/datacart/rpa", "headbagCacheSize": 50000000, "bagstoreMode": "local", - "jwtSecretKey": "1234567890 pdr.rpa.2023 1234567890" + "jwtSecretKey": "1234567890 pdr.rpa.2023 1234567890", + "expiresAfterMillis": 1209600000 } From de80e31caf478446a8978851e1406ad838b9608d Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 11:15:12 -0500 Subject: [PATCH 10/10] change 'expiresIn' metadata field name to 'expires' --- .../distrib/cachemgr/CacheExpiryCheck.java | 15 +++--- .../pdr/RestrictedDatasetRestorer.java | 4 +- .../cachemgr/CacheExpiryCheckTest.java | 46 +++++++++---------- .../pdr/RestrictedDatasetRestorerTest.java | 36 +++++++-------- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index c5ef1845..2a8f718a 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -12,7 +12,6 @@ */ public class CacheExpiryCheck implements CacheObjectCheck { - // private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds private StorageInventoryDB inventoryDB; public CacheExpiryCheck(StorageInventoryDB inventoryDB) { @@ -21,8 +20,8 @@ public CacheExpiryCheck(StorageInventoryDB inventoryDB) { /** * Checks if a cache object is expired and removes it from the cache if it is. - * The method uses the {@code expiresIn} metadata field to determine the expiration status. - * The expiration time is calculated based on the {@code LastModified} time plus the {@code expiresIn} duration. + * The method uses the {@code expires} metadata field to determine the expiration status. + * The expiration time is calculated based on the {@code LastModified} time plus the {@code expires} duration. * If the current time is past the calculated expiry time, the object is removed from the inventory database. * * @param co The cache object to check for expiration. @@ -36,10 +35,10 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); } - if (co.hasMetadatum("expiresIn")) { - long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); - if (expiresInDuration == -1L) { - throw new IntegrityException("Invalid 'expiresIn' metadata value"); + if (co.hasMetadatum("expires")) { + long expiresDuration = co.getMetadatumLong("expires", -1L); + if (expiresDuration == -1L) { + throw new IntegrityException("Invalid 'expires' metadata value"); } long lastModified = co.getLastModified(); @@ -47,7 +46,7 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IntegrityException("CacheObject 'lastModified' time not available"); } - long expiryTime = lastModified + expiresInDuration; + long expiryTime = lastModified + expiresDuration; long currentTime = Instant.now().toEpochMilli(); // Check if the object is expired diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java index d06dff33..54bec7ab 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java @@ -295,8 +295,8 @@ protected void cacheFromBag(String bagfile, Collection need, Collection< protected void updateMetadata(JSONObject md, int prefs) { if ((prefs & ROLE_RESTRICTED_DATA) != 0) { // Calculate the expiration time as current time + expiryTime - long expiresIn = System.currentTimeMillis() + expiryTime; - md.put("expiresIn", expiresIn); + long expires = System.currentTimeMillis() + expiryTime; + md.put("expires", expires); } } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index 6796b053..0f0c61da 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -30,7 +30,7 @@ public void setUp() { /** * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. - * An object is considered expired based on the `expiresIn` metadata, which defines the duration after which + * An object is considered expired based on the `expires` metadata, which defines the duration after which * an object should be considered expired from the time of its last modification. This test ensures that an object * past its expiration is appropriately removed from the inventory database. * @@ -40,8 +40,8 @@ public void setUp() { public void testExpiredObjectRemoval() throws Exception { // Setup an expired cache object cacheObject.volume = mockVolume; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second when(cacheObject.getLastModified()).thenReturn(Instant.now().minusSeconds(10).toEpochMilli()); when(cacheObject.volume.remove(cacheObject.name)).thenReturn(true); @@ -54,7 +54,7 @@ public void testExpiredObjectRemoval() throws Exception { /** * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if the current time has not - * exceeded its `expiresIn` duration since its last modification. This test verifies that no removal action is taken + * exceeded its `expires` duration since its last modification. This test verifies that no removal action is taken * for such non-expired objects. * * @throws Exception to handle any exceptions thrown during the test execution @@ -64,8 +64,8 @@ public void testNonExpiredObject() throws Exception { // Setup mock cacheObject.name = "nonExpiredObject"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds long lastModified = System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000L); // 7 days ago, within expiry period when(cacheObject.getLastModified()).thenReturn(lastModified); @@ -77,16 +77,16 @@ public void testNonExpiredObject() throws Exception { } /** - * Tests that no action is taken and no exception is thrown for a cache object without the {@code expiresIn} metadata. - * This verifies that the absence of {@code expiresIn} metadata does not trigger any removal process or result in an error. + * Tests that no action is taken and no exception is thrown for a cache object without the {@code expires} metadata. + * This verifies that the absence of {@code expires} metadata does not trigger any removal process or result in an error. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test - public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception { - cacheObject.name = "objectWithoutExpiresIn"; + public void testObjectWithoutExpiresMetadata_NoActionTaken() throws Exception { + cacheObject.name = "objectWithoutExpires"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + when(cacheObject.hasMetadatum("expires")).thenReturn(false); expiryCheck.check(cacheObject); @@ -96,15 +96,15 @@ public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception /** * Test to ensure that a cache object with an expiration date in the future is not removed from the cache. * This test verifies the {@code check} method's correct behavior in handling non-expired objects based - * on the {@code expiresIn} metadata. + * on the {@code expires} metadata. * * @throws Exception to handle any exceptions thrown during the test execution. */ @Test public void testNonExpiredObject_NoRemoval() throws Exception { // Setup a non-expired cache object - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future when(cacheObject.getLastModified()).thenReturn(System.currentTimeMillis()); expiryCheck.check(cacheObject); @@ -115,32 +115,32 @@ public void testNonExpiredObject_NoRemoval() throws Exception { /** - * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expiresIn} metadata + * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expires} metadata * but lacks a valid {@code lastModified} time. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test(expected = IntegrityException.class) - public void testObjectWithExpiresInButNoLastModified_ThrowsException() throws Exception { + public void testObjectWithExpiresButNoLastModified_ThrowsException() throws Exception { cacheObject.name = "objectWithNoLastModified"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second when(cacheObject.getLastModified()).thenReturn(-1L); // Last modified not available expiryCheck.check(cacheObject); } /** - * Test to verify that no action is taken for a cache object missing the {@code expiresIn} metadata. - * This test ensures that the absence of {@code expiresIn} metadata does not trigger any removal or error. + * Test to verify that no action is taken for a cache object missing the {@code expires} metadata. + * This test ensures that the absence of {@code expires} metadata does not trigger any removal or error. * * @throws Exception to handle any exceptions thrown during the test execution. */ @Test - public void testObjectWithoutExpiresIn_NoAction() throws Exception { - // Setup an object without expiresIn metadata - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + public void testObjectWithoutExpires_NoAction() throws Exception { + // Setup an object without expires metadata + when(cacheObject.hasMetadatum("expires")).thenReturn(false); expiryCheck.check(cacheObject); diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java index eff08566..2de7d871 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java @@ -407,7 +407,7 @@ public void testCacheFromBag() } @Test - public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { + public void testExpires() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { assertTrue(! cache.isCached("mds1491/trial1.json")); assertTrue(! cache.isCached("mds1491/trial2.json")); @@ -429,33 +429,33 @@ public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundExcep List found = cache.getInventoryDB().findObject("mds1491/trial1.json", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - // Verify the "expiresIn" value is approximately 2 weeks from the current time + // Verify the "expires" value is approximately 2 weeks from the current time long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; - long expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - long actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - // Check that the absolute difference between the expected and actual expiresIn values is less than 1000ms - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + long expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + long actualExpires = found.get(0).getMetadatumLong("expires", 0); + // Check that the absolute difference between the expected and actual expires values is less than 1000ms + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); found = cache.getInventoryDB().findObject("mds1491/trial2.json", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + expectedExpires= System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); found = cache.getInventoryDB().findObject("mds1491/README.txt", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); }