From 170e0a721b16d8e8af101bbe92d28fe1cbd0d4c9 Mon Sep 17 00:00:00 2001 From: Arafat2198 <98023601+ArafatKhan2198@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:03:53 +0530 Subject: [PATCH 01/51] HDDS-9721. Intermittent timeout in TestReconInsightsForDeletedDirectories (#5647) --- ...estReconInsightsForDeletedDirectories.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java index 66a5b9907ca..ce89f8ffe41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java @@ -88,9 +88,9 @@ public class TestReconInsightsForDeletedDirectories { @BeforeAll public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_DIR_DELETING_SERVICE_INTERVAL, 2000); - conf.setInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, 5); - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, + conf.setInt(OZONE_DIR_DELETING_SERVICE_INTERVAL, 1000000); + conf.setInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, 0); + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 10000000, TimeUnit.MILLISECONDS); conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); conf.setBoolean(OZONE_ACL_ENABLED, true); @@ -176,9 +176,7 @@ public void testGetDeletedDirectoryInfo() assertTableRowCount(omDirTable, 1, false); // Sync data from Ozone Manager to Recon. - OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); - impl.syncDataFromOM(); + syncDataFromOM(); // Retrieve tables from Recon's OM-DB. ReconOMMetadataManager reconOmMetadataManagerInstance = @@ -222,10 +220,10 @@ public void testGetDeletedDirectoryInfo() // Delete the entire directory dir1. fs.delete(dir1, true); - - impl.syncDataFromOM(); + syncDataFromOM(); // Check the count of recon directory table and recon deletedDirectory table assertTableRowCount(reconDirTable, 0, true); + assertTableRowCount(reconDeletedDirTable, 1, true); // Create an Instance of OMDBInsightEndpoint. @@ -243,7 +241,7 @@ public void testGetDeletedDirectoryInfo() Response deletedDirInfo = omdbInsightEndpoint.getDeletedDirInfo(-1, ""); KeyInsightInfoResponse entity = (KeyInsightInfoResponse) deletedDirInfo.getEntity(); - // Assert the size of deleted directory is 700. + // Assert the size of deleted directory is 10. Assert.assertEquals(10, entity.getUnreplicatedDataSize()); // Cleanup the tables. @@ -290,9 +288,7 @@ public void testGetDeletedDirectoryInfoForNestedDirectories() assertTableRowCount(omDirTable, 3, false); // Sync data from Ozone Manager to Recon. - OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); - impl.syncDataFromOM(); + syncDataFromOM(); // Retrieve tables from Recon's OM-DB. ReconOMMetadataManager reconOmMetadataManagerInstance = @@ -323,7 +319,8 @@ public void testGetDeletedDirectoryInfoForNestedDirectories() // Delete the entire root directory dir1. fs.delete(new Path("/dir1/dir2/dir3"), true); - impl.syncDataFromOM(); + syncDataFromOM(); + // Verify the entries in the Recon tables after sync. assertTableRowCount(reconFileTable, 3, true); assertTableRowCount(reconDirTable, 2, true); @@ -333,7 +330,7 @@ public void testGetDeletedDirectoryInfoForNestedDirectories() Response deletedDirInfo = omdbInsightEndpoint.getDeletedDirInfo(-1, ""); KeyInsightInfoResponse entity = (KeyInsightInfoResponse) deletedDirInfo.getEntity(); - // Assert the size of deleted directory is 1000. + // Assert the size of deleted directory is 3. Assert.assertEquals(3, entity.getUnreplicatedDataSize()); // Cleanup the tables. @@ -378,9 +375,8 @@ public void testGetDeletedDirectoryInfoWithMultipleSubdirectories() Assertions.assertFalse(fs.exists(rootDir), "Directory was not deleted"); // Sync data from Ozone Manager to Recon. - OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); - impl.syncDataFromOM(); + syncDataFromOM(); + // Fetch the deleted directory info from Recon OmDbInsightEndpoint. OzoneStorageContainerManager reconSCM = cluster.getReconServer().getReconStorageContainerManager(); @@ -484,6 +480,13 @@ private boolean assertTableRowCount(int expectedCount, return count == expectedCount; } + private void syncDataFromOM() { + // Sync data from Ozone Manager to Recon. + OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) + cluster.getReconServer().getOzoneManagerServiceProvider(); + impl.syncDataFromOM(); + } + private static BucketLayout getFSOBucketLayout() { return BucketLayout.FILE_SYSTEM_OPTIMIZED; From 7dcae6715cc449b5ffd41b31fdda38fa0560dcc3 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Mon, 27 Nov 2023 11:54:31 +0000 Subject: [PATCH 02/51] HDDS-9593. Replication Manager: Do not count unique origin nodes as over-replicated (#5592) --- .../replication/ReplicationManager.java | 2 +- .../health/RatisReplicationCheckHandler.java | 49 +++++++++- .../TestRatisReplicationCheckHandler.java | 97 ++++++++++++++++++- 3 files changed, 141 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java index 34b0183ffc5..d34d14ad210 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java @@ -243,7 +243,7 @@ public ReplicationManager(final ConfigurationSource conf, this.ecMisReplicationCheckHandler = new ECMisReplicationCheckHandler(ecContainerPlacement); this.ratisReplicationCheckHandler = - new RatisReplicationCheckHandler(ratisContainerPlacement); + new RatisReplicationCheckHandler(ratisContainerPlacement, this); this.nodeManager = nodeManager; this.metrics = ReplicationManagerMetrics.create(this); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java index a1d54ad1fce..40531cb04fc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.container.replication.health; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -26,6 +27,9 @@ import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; import org.apache.hadoop.hdds.scm.container.replication.RatisContainerReplicaCount; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManagerUtil; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,9 +56,12 @@ public class RatisReplicationCheckHandler extends AbstractCheck { * should be replicated. */ private final PlacementPolicy ratisContainerPlacement; + private final ReplicationManager replicationManager; - public RatisReplicationCheckHandler(PlacementPolicy containerPlacement) { + public RatisReplicationCheckHandler(PlacementPolicy containerPlacement, + ReplicationManager replicationManager) { this.ratisContainerPlacement = containerPlacement; + this.replicationManager = replicationManager; } @Override @@ -190,6 +197,12 @@ public ContainerHealthResult checkHealth(ContainerCheckRequest request) { return replicaCount.toUnderHealthResult(); } + + if (replicaCount.isOverReplicated(false)) { + // If the container is over replicated without considering UNHEALTHY + // then we know for sure it is over replicated, so mark as such. + return replicaCount.toOverHealthResult(); + } /* When checking for over replication, consider UNHEALTHY replicas. This means that other than checking over replication of healthy replicas (such as 4 @@ -200,9 +213,37 @@ of UNHEALTHY replicas (such as 3 CLOSED and 1 UNHEALTHY replicas of a RatisContainerReplicaCount consideringUnhealthy = new RatisContainerReplicaCount(container, replicas, replicaPendingOps, minReplicasForMaintenance, true); - boolean isOverReplicated = consideringUnhealthy.isOverReplicated(false); - if (isOverReplicated) { - return consideringUnhealthy.toOverHealthResult(); + + if (consideringUnhealthy.isOverReplicated(false)) { + if (container.getState() == HddsProtos.LifeCycleState.CLOSED) { + return consideringUnhealthy.toOverHealthResult(); + } else if (container.getState() + == HddsProtos.LifeCycleState.QUASI_CLOSED) { + // If the container is quasi-closed and over replicated, we may have a + // case where the excess replica is an unhealthy one, but it has a + // unique origin and therefore should not be deleted. In this case, + // we should not mark the container as over replicated. + // We ignore pending deletes, as a container is still over replicated + // until the pending delete completes. + ContainerReplica toDelete = ReplicationManagerUtil + .selectUnhealthyReplicaForDelete(container, replicas, 0, + (dnd) -> { + try { + return replicationManager.getNodeStatus(dnd); + } catch (NodeNotFoundException e) { + return null; + } + }); + if (toDelete != null) { + // There is at least one unhealthy replica that can be deleted, so + // return as over replicated. + return consideringUnhealthy.toOverHealthResult(); + } else { + // Even though we have at least 4 replicas with some unhealthy, we + // can't delete any of them, so the container is not over replicated. + return new ContainerHealthResult.HealthyResult(container); + } + } } int requiredNodes = container.getReplicationConfig().getRequiredNodes(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java index e2de8919667..ad3f3fef6d7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java @@ -36,8 +36,11 @@ import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult.OverReplicatedHealthResult; import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult.UnderReplicatedHealthResult; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; import org.apache.hadoop.hdds.scm.container.replication.ReplicationQueue; import org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -50,6 +53,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; @@ -76,17 +80,23 @@ public class TestRatisReplicationCheckHandler { private ReplicationQueue repQueue; private ContainerCheckRequest.Builder requestBuilder; private ReplicationManagerReport report; + private ReplicationManager replicationManager; private int maintenanceRedundancy = 2; @BeforeEach - public void setup() throws IOException { + public void setup() throws IOException, NodeNotFoundException { containerPlacementPolicy = Mockito.mock(PlacementPolicy.class); Mockito.when(containerPlacementPolicy.validateContainerPlacement( Mockito.any(), Mockito.anyInt() )).thenAnswer(invocation -> new ContainerPlacementStatusDefault(2, 2, 3)); - healthCheck = new RatisReplicationCheckHandler(containerPlacementPolicy); + + replicationManager = Mockito.mock(ReplicationManager.class); + Mockito.when(replicationManager.getNodeStatus(Mockito.any())) + .thenReturn(NodeStatus.inServiceHealthy()); + healthCheck = new RatisReplicationCheckHandler(containerPlacementPolicy, + replicationManager); repConfig = RatisReplicationConfig.getInstance(THREE); repQueue = new ReplicationQueue(); report = new ReplicationManagerReport(); @@ -892,4 +902,87 @@ public void testWithQuasiClosedReplicasWithWrongSequenceID() { ReplicationManagerReport.HealthState.UNDER_REPLICATED)); } + @Test + public void testExcessReplicasButNotOverReplicatedDuetoUniqueOrigins() { + final long sequenceID = 20; + final ContainerInfo container = ReplicationTestUtil.createContainerInfo( + repConfig, 1, HddsProtos.LifeCycleState.QUASI_CLOSED, + sequenceID); + + final Set replicas = new HashSet<>(2); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID - 1)); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID - 1)); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID - 1)); + + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.UNHEALTHY, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + + assertFalse(healthCheck.handle(requestBuilder.build())); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(0, repQueue.overReplicatedQueueSize()); + assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + + @Test + public void testExcessReplicasAndOverReplicatedDuetoNonUniqueOrigins() { + final long sequenceID = 20; + final ContainerInfo container = ReplicationTestUtil.createContainerInfo( + repConfig, 1, HddsProtos.LifeCycleState.QUASI_CLOSED, + sequenceID); + + UUID origin = UUID.randomUUID(); + final Set replicas = new HashSet<>(2); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID - 1)); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails().getUuid(), + sequenceID - 1)); + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), origin, + sequenceID - 1)); + + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.UNHEALTHY, 1, 1, + MockDatanodeDetails.randomDatanodeDetails(), origin, + sequenceID - 1)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + + assertTrue(healthCheck.handle(requestBuilder.build())); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(1, repQueue.overReplicatedQueueSize()); + assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + } From ffe2a2e53bea60863d1517d20c3bf4a7b9c4c043 Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Mon, 27 Nov 2023 18:02:36 +0530 Subject: [PATCH 03/51] HDDS-9737. Legacy Replication Manager should consider that UNHEALTHY replicas might be decommissioning (#5674) --- .../LegacyRatisContainerReplicaCount.java | 8 +++ .../replication/LegacyReplicationManager.java | 26 ++++++- .../TestLegacyReplicationManager.java | 29 ++++++++ .../scm/node/TestDatanodeAdminMonitor.java | 67 +++++++++++++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java index 90c7afebcce..f708ae1ead9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.node.NodeManager; +import java.util.List; import java.util.Set; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; @@ -50,6 +51,13 @@ public LegacyRatisContainerReplicaCount(ContainerInfo container, minHealthyForMaintenance); } + public LegacyRatisContainerReplicaCount(ContainerInfo container, + Set replicas, List pendingOps, + int minHealthyForMaintenance, boolean considerUnhealthy) { + super(container, replicas, pendingOps, minHealthyForMaintenance, + considerUnhealthy); + } + @Override protected int healthyReplicaCountAdapter() { return -getMisMatchedReplicaCount(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java index 9a51537028b..07a8f730ec0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java @@ -1051,7 +1051,7 @@ public ContainerReplicaCount getContainerReplicaCount(ContainerInfo container) synchronized (container) { final Set replica = containerManager .getContainerReplicas(container.containerID()); - return getContainerReplicaCount(container, replica); + return getReplicaCountOptionallyConsiderUnhealthy(container, replica); } } @@ -1076,6 +1076,28 @@ private RatisContainerReplicaCount getContainerReplicaCount( minHealthyForMaintenance); } + private RatisContainerReplicaCount getReplicaCountOptionallyConsiderUnhealthy( + ContainerInfo container, Set replicas) { + LegacyRatisContainerReplicaCount withUnhealthy = + new LegacyRatisContainerReplicaCount(container, replicas, + getPendingOps(container.containerID()), minHealthyForMaintenance, + true); + if (withUnhealthy.getHealthyReplicaCount() == 0 && + withUnhealthy.getUnhealthyReplicaCount() > 0) { + // if the container has only UNHEALTHY replicas, return the correct + // RatisContainerReplicaCount object which can handle UNHEALTHY replicas + return withUnhealthy; + } + + return new LegacyRatisContainerReplicaCount( + container, + replicas, + getInflightAdd(container.containerID()), + getInflightDel(container.containerID()), + container.getReplicationConfig().getRequiredNodes(), + minHealthyForMaintenance); + } + /** * Returns true if more than 50% of the container replicas with unique * originNodeId are in QUASI_CLOSED state. @@ -1345,7 +1367,7 @@ private void handleAllReplicasUnhealthy(ContainerInfo container, List replicas = replicaSet.getReplicas(); RatisContainerReplicaCount unhealthyReplicaSet = - new RatisContainerReplicaCount(container, + new LegacyRatisContainerReplicaCount(container, new HashSet<>(replicaSet.getReplicas()), getPendingOps(container.containerID()), minHealthyForMaintenance, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java index 43a2fb32633..c9bd4bddbda 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestLegacyReplicationManager.java @@ -312,6 +312,35 @@ public void testReplicationManagerRestart() throws InterruptedException { replicationManager.start(); Assertions.assertTrue(replicationManager.isRunning()); } + + @Test + public void testGetContainerReplicaCount() + throws IOException, TimeoutException { + ContainerInfo container = createContainer(LifeCycleState.QUASI_CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), UNHEALTHY); + addReplica(container, NodeStatus.inServiceHealthy(), UNHEALTHY); + ContainerReplica decommissioningReplica = + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), + UNHEALTHY); + + ContainerReplicaCount replicaCount = + replicationManager.getLegacyReplicationManager() + .getContainerReplicaCount(container); + + Assertions.assertTrue( + replicaCount instanceof LegacyRatisContainerReplicaCount); + Assertions.assertFalse(replicaCount.isSufficientlyReplicated()); + Assertions.assertFalse(replicaCount.isSufficientlyReplicatedForOffline( + decommissioningReplica.getDatanodeDetails(), nodeManager)); + + addReplica(container, NodeStatus.inServiceHealthy(), UNHEALTHY); + replicaCount = replicationManager.getLegacyReplicationManager() + .getContainerReplicaCount(container); + Assertions.assertTrue(replicaCount.isSufficientlyReplicated()); + Assertions.assertTrue(replicaCount.isSufficientlyReplicatedForOffline( + decommissioningReplica.getDatanodeDetails(), nodeManager)); + Assertions.assertTrue(replicaCount.isHealthyEnoughForOffline()); + } } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 48afbe06820..09ff7cb0ff6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -42,6 +42,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -295,6 +296,72 @@ public void testDecommissionWaitsForUnhealthyReplicaToReplicate() nodeManager.getNodeStatus(dn1).getOperationalState()); } + /** + * Consider a QUASI_CLOSED container with only UNHEALTHY replicas. If one + * of its nodes is decommissioned, the decommissioning should succeed. + */ + @Test + public void testQuasiClosedContainerWithAllUnhealthyReplicas() + throws NodeNotFoundException, ContainerNotFoundException { + conf.setBoolean("hdds.scm.replication.enable.legacy", true); + + DatanodeDetails decommissioningNode = + MockDatanodeDetails.randomDatanodeDetails(); + nodeManager.register(decommissioningNode, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); + ContainerInfo container = ReplicationTestUtil.createContainer( + HddsProtos.LifeCycleState.QUASI_CLOSED, + RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)); + Set replicas = + ReplicationTestUtil.createReplicas(container.containerID(), + State.UNHEALTHY, 0, 0); + + ContainerReplica decommissioningReplica = + ReplicationTestUtil.createContainerReplica(container.containerID(), 0, + DECOMMISSIONING, State.UNHEALTHY, container.getNumberOfKeys(), + container.getUsedBytes(), decommissioningNode, + decommissioningNode.getUuid()); + replicas.add(decommissioningReplica); + nodeManager.setContainers(decommissioningNode, + ImmutableSet.of(container.containerID())); + + Mockito.when(repManager.getContainerReplicaCount( + Mockito.eq(container.containerID()))) + .thenReturn(new LegacyRatisContainerReplicaCount(container, replicas, + Collections.emptyList(), 2, true)); + + // start monitoring dn1 + monitor.startMonitoring(decommissioningNode); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(decommissioningNode).getOperationalState()); + + // Running the monitor again causes it to remain DECOMMISSIONING + // as nothing has changed. + monitor.run(); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(decommissioningNode).getOperationalState()); + + // add a copy of the UNHEALTHY replica on a new node, decommissioningNode + // should get decommissioned now + ContainerReplica copyOfUnhealthyOnNewNode = + decommissioningReplica.toBuilder() + .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) + .build(); + replicas.add(copyOfUnhealthyOnNewNode); + Mockito.when(repManager.getContainerReplicaCount( + Mockito.eq(container.containerID()))) + .thenReturn(new LegacyRatisContainerReplicaCount(container, replicas, + Collections.emptyList(), 3, true)); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + nodeManager.getNodeStatus(decommissioningNode).getOperationalState()); + } + @Test public void testDecommissionNotBlockedByDeletingContainers() throws NodeNotFoundException, ContainerNotFoundException { From c0e49b90fb2a79010fe734158b2d494a9ad20680 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Mon, 27 Nov 2023 19:59:36 +0530 Subject: [PATCH 04/51] HDDS-3506. Enable TestOzoneFileInterfaces#testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile (#5675) --- .../org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java index 50269557932..701dffa4b97 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -63,8 +63,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; -import org.apache.ozone.test.UnhealthyTest; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -78,7 +76,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; import org.junit.rules.Timeout; import org.apache.ozone.test.JUnit5AwareTimeout; @@ -518,7 +515,6 @@ public void testOzoneManagerLocatedFileStatus() throws IOException { } @Test - @Category(UnhealthyTest.class) @Unhealthy("HDDS-3506") public void testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile() throws Exception { // naive assumption: MiniOzoneCluster will not have larger than ~1GB From aeb504029d280ca5c405784c5e5f74c208c904f8 Mon Sep 17 00:00:00 2001 From: Raju Balpande <146973984+raju-balpande@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:15:32 +0530 Subject: [PATCH 05/51] HDDS-9691. Migrate remaining tests in ozone-recon to JUnit5 (#5669) --- hadoop-ozone/recon/pom.xml | 6 +- .../recon/OMMetadataManagerTestUtils.java | 2 +- .../hadoop/ozone/recon/TestReconUtils.java | 54 +++--- .../ozone/recon/common/CommonUtils.java | 119 +++++------- .../TestReconWithDifferentSqlDBs.java | 4 +- .../ozone/recon/scm/TestReconNodeManager.java | 8 +- .../TestOzoneManagerServiceProviderImpl.java | 179 +++++++++--------- .../recon/spi/impl/TestReconDBProvider.java | 2 +- .../ozone/recon/tasks/TestNSSummaryTask.java | 64 ++++--- .../recon/tasks/TestNSSummaryTaskWithFSO.java | 104 +++++----- .../tasks/TestNSSummaryTaskWithLegacy.java | 108 +++++------ 11 files changed, 313 insertions(+), 337 deletions(-) diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 933ac426e15..3bbe8af1bbd 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -25,6 +25,7 @@ ozone-recon 7.33.6 + false @@ -300,11 +301,6 @@ test-jar test - - junit - junit - test - org.junit.jupiter junit-jupiter-params diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index cadfc5ca9b5..42d69e030f3 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -27,7 +27,7 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.VOLUME_TABLE; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java index b138b934fb2..b34c8d31c6f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java @@ -20,8 +20,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.recon.ReconUtils.createTarFile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -36,6 +37,8 @@ import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.net.URL; import java.util.Random; @@ -44,46 +47,37 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdfs.web.URLConnectionFactory; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; /** * Test Recon Utility methods. */ public class TestReconUtils { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + @TempDir + private Path temporaryFolder; @Test public void testGetReconDbDir() throws Exception { - String filePath = folder.getRoot().getAbsolutePath(); OzoneConfiguration configuration = new OzoneConfiguration(); - configuration.set("TEST_DB_DIR", filePath); + configuration.set("TEST_DB_DIR", temporaryFolder.toString()); File file = new ReconUtils().getReconDbDir(configuration, "TEST_DB_DIR"); - Assert.assertEquals(filePath, file.getAbsolutePath()); + assertEquals(temporaryFolder.toString(), file.getAbsolutePath()); } @Test - public void testCreateTarFile() throws Exception { + public void testCreateTarFile(@TempDir File tempSnapshotDir) + throws Exception { - File tempSnapshotDir = null; FileInputStream fis = null; FileOutputStream fos = null; File tarFile = null; try { - String testDirName = System.getProperty("java.io.tmpdir"); - if (!testDirName.endsWith("/")) { - testDirName += "/"; - } - testDirName += "TestCreateTarFile_Dir" + System.currentTimeMillis(); - tempSnapshotDir = new File(testDirName); - tempSnapshotDir.mkdirs(); + String testDirName = tempSnapshotDir.getPath(); File file = new File(testDirName + "/temp1.txt"); OutputStreamWriter writer = new OutputStreamWriter( @@ -98,7 +92,7 @@ public void testCreateTarFile() throws Exception { writer.close(); tarFile = createTarFile(Paths.get(testDirName)); - Assert.assertNotNull(tarFile); + assertNotNull(tarFile); } finally { org.apache.hadoop.io.IOUtils.closeStream(fis); @@ -111,8 +105,8 @@ public void testCreateTarFile() throws Exception { @Test public void testUntarCheckpointFile() throws Exception { - File newDir = folder.newFolder(); - + File newDir = Files.createDirectory( + temporaryFolder.resolve("NewDir")).toFile(); File file1 = Paths.get(newDir.getAbsolutePath(), "file1") .toFile(); String str = "File1 Contents"; @@ -131,17 +125,18 @@ public void testUntarCheckpointFile() throws Exception { //Create test tar file. File tarFile = createTarFile(newDir.toPath()); - File outputDir = folder.newFolder(); + File outputDir = Files.createDirectory( + temporaryFolder.resolve("OutputDir")).toFile(); new ReconUtils().untarCheckpointFile(tarFile, outputDir.toPath()); assertTrue(outputDir.isDirectory()); - assertTrue(outputDir.listFiles().length == 2); + assertEquals(2, outputDir.listFiles().length); } @Test public void testMakeHttpCall() throws Exception { String url = "http://localhost:9874/dbCheckpoint"; - File file1 = Paths.get(folder.getRoot().getPath(), "file1") + File file1 = Paths.get(temporaryFolder.toString(), "file1") .toFile(); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(file1.getAbsoluteFile()), UTF_8)); @@ -165,8 +160,7 @@ public void testMakeHttpCall() throws Exception { } @Test - public void testGetLastKnownDB() throws IOException { - File newDir = folder.newFolder(); + public void testGetLastKnownDB(@TempDir File newDir) throws IOException { File file1 = Paths.get(newDir.getAbsolutePath(), "valid_1") .toFile(); @@ -195,7 +189,7 @@ public void testGetLastKnownDB() throws IOException { ReconUtils reconUtils = new ReconUtils(); File latestValidFile = reconUtils.getLastKnownDB(newDir, "valid"); - assertTrue(latestValidFile.getName().equals("valid_2")); + assertEquals("valid_2", latestValidFile.getName()); } @Test @@ -221,7 +215,7 @@ public void testNextClosestPowerIndexOfTwo() { static void assertNextClosestPowerIndexOfTwo(long n) { final int expected = oldNextClosestPowerIndexOfTwoFixed(n); final int computed = ReconUtils.nextClosestPowerIndexOfTwo(n); - Assert.assertEquals("n=" + n, expected, computed); + assertEquals(expected, computed, "n=" + n); } private static int oldNextClosestPowerIndexOfTwoFixed(long n) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java index fc3fcf813b7..35196a748b5 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java @@ -34,7 +34,6 @@ import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; -import org.junit.Assert; import javax.ws.rs.core.Response; @@ -43,6 +42,7 @@ import java.util.HashMap; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * This is a utility class for common code for test cases. @@ -84,21 +84,20 @@ public void testNSSummaryBasicInfoRoot( Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); NamespaceSummaryResponse rootResponseObj = (NamespaceSummaryResponse) rootResponse.getEntity(); - Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); - Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("USER", + assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); + assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); + assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); + assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); + assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); + assertEquals("USER", rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); - Assert.assertEquals("WRITE", - rootResponseObj.getObjectDBInfo().getAcls().get(0) + assertEquals("WRITE", rootResponseObj.getObjectDBInfo().getAcls().get(0) .getAclList().get(0)); - Assert.assertEquals(username, + assertEquals(username, rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); - Assert.assertEquals("value", + assertEquals("value", rootResponseObj.getObjectDBInfo().getMetadata().get("key")); - Assert.assertEquals("ACCESS", + assertEquals("ACCESS", rootResponseObj.getObjectDBInfo().getAcls().get(0).getScope()); } @@ -107,26 +106,18 @@ public void testNSSummaryBasicInfoVolume( Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); NamespaceSummaryResponse volResponseObj = (NamespaceSummaryResponse) volResponse.getEntity(); - Assert.assertEquals(EntityType.VOLUME, + assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); - Assert.assertEquals(2, - volResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(4, - volResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(6, - volResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj. + assertEquals(2, volResponseObj.getCountStats().getNumBucket()); + assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); + assertEquals(6, volResponseObj.getCountStats().getNumTotalKey()); + assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. getObjectDBInfo()).getAdmin()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj. + assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. getObjectDBInfo()).getOwner()); - Assert.assertEquals("vol", - volResponseObj.getObjectDBInfo().getName()); - Assert.assertEquals(2097152, - volResponseObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(-1, - volResponseObj.getObjectDBInfo().getQuotaInNamespace()); + assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); + assertEquals(2097152, volResponseObj.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, volResponseObj.getObjectDBInfo().getQuotaInNamespace()); } public void testNSSummaryBasicInfoBucketOne(BucketLayout bucketLayout, @@ -135,18 +126,18 @@ public void testNSSummaryBasicInfoBucketOne(BucketLayout bucketLayout, nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); NamespaceSummaryResponse bucketOneObj = (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", + assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); + assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); + assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); + assertEquals("vol", ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, + assertEquals(StorageType.DISK, ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(bucketLayout, + assertEquals(bucketLayout, ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket1", + assertEquals("bucket1", ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); } @@ -157,18 +148,18 @@ public void testNSSummaryBasicInfoBucketTwo( nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); NamespaceSummaryResponse bucketTwoObj = (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", + assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); + assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); + assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); + assertEquals("vol", ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, + assertEquals(StorageType.DISK, ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(bucketLayout, + assertEquals(bucketLayout, ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket2", + assertEquals("bucket2", ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); } @@ -177,21 +168,14 @@ public void testNSSummaryBasicInfoDir( Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); NamespaceSummaryResponse dirOneObj = (NamespaceSummaryResponse) dirOneResponse.getEntity(); - Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - Assert.assertEquals(3, - dirOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(3, - dirOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("dir1", - dirOneObj.getObjectDBInfo().getName()); - Assert.assertEquals(0, - dirOneObj.getObjectDBInfo().getMetadata().size()); - Assert.assertEquals(0, - dirOneObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(0, - dirOneObj.getObjectDBInfo().getQuotaInNamespace()); - Assert.assertEquals(0, - dirOneObj.getObjectDBInfo().getUsedNamespace()); + assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); + assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); + assertEquals(3, dirOneObj.getCountStats().getNumTotalKey()); + assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); + assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); + assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); + assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); + assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); } public void testNSSummaryBasicInfoNoPath( @@ -200,10 +184,9 @@ public void testNSSummaryBasicInfoNoPath( .getBasicInfo(INVALID_PATH); NamespaceSummaryResponse invalidObj = (NamespaceSummaryResponse) invalidResponse.getEntity(); - Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - Assert.assertEquals(null, invalidObj.getCountStats()); - Assert.assertEquals(null, invalidObj.getObjectDBInfo()); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + assertEquals(null, invalidObj.getCountStats()); + assertEquals(null, invalidObj.getObjectDBInfo()); } public void testNSSummaryBasicInfoKey( @@ -211,16 +194,16 @@ public void testNSSummaryBasicInfoKey( Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); NamespaceSummaryResponse keyResObj = (NamespaceSummaryResponse) keyResponse.getEntity(); - Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); - Assert.assertEquals("vol", + assertEquals(EntityType.KEY, keyResObj.getEntityType()); + assertEquals("vol", ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals("bucket2", + assertEquals("bucket2", ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); - Assert.assertEquals("file4", + assertEquals("file4", ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); - Assert.assertEquals(2049, + assertEquals(2049, ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); - Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, + assertEquals(HddsProtos.ReplicationType.STAND_ALONE, ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). getReplicationConfig().getReplicationType()); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java index 721e524dade..b4cf7689572 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java @@ -20,8 +20,8 @@ import static org.apache.hadoop.ozone.recon.ReconControllerModule.ReconDaoBindingModule.RECON_DAO_LIST; import static org.hadoop.ozone.recon.codegen.SqlDbUtils.SQLITE_DRIVER_CLASS; import static org.hadoop.ozone.recon.schema.Tables.RECON_TASK_STATUS; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.File; import java.io.IOException; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java index 4f6b09309a0..2e52b707df0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java @@ -24,10 +24,10 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java index e93073cbd0a..497d05c4f80 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java @@ -30,10 +30,11 @@ import static org.apache.hadoop.ozone.recon.ReconUtils.createTarFile; import static org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl.OmSnapshotTaskName.OmDeltaRequest; import static org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl.OmSnapshotTaskName.OmSnapshotRequest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -75,11 +76,9 @@ import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao; import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; import org.rocksdb.TransactionLogIterator.BatchResult; import org.rocksdb.WriteBatch; @@ -89,33 +88,33 @@ */ public class TestOzoneManagerServiceProviderImpl { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private OzoneConfiguration configuration; private OzoneManagerProtocol ozoneManagerProtocol; private CommonUtils commonUtils; - @Before - public void setUp() throws Exception { + @BeforeEach + public void setUp(@TempDir File dirReconSnapDB, @TempDir File dirReconDB) + throws Exception { configuration = new OzoneConfiguration(); configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, - temporaryFolder.newFolder().getAbsolutePath()); + dirReconSnapDB.getAbsolutePath()); configuration.set(OZONE_RECON_DB_DIR, - temporaryFolder.newFolder().getAbsolutePath()); + dirReconDB.getAbsolutePath()); configuration.set("ozone.om.address", "localhost:9862"); ozoneManagerProtocol = getMockOzoneManagerClient(new DBUpdates()); commonUtils = new CommonUtils(); } @Test - public void testUpdateReconOmDBWithNewSnapshot() throws Exception { + public void testUpdateReconOmDBWithNewSnapshot( + @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) + throws Exception { OMMetadataManager omMetadataManager = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata); ReconOMMetadataManager reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - temporaryFolder.newFolder()); + dirReconMetadata); writeDataToOm(omMetadataManager, "key_one"); writeDataToOm(omMetadataManager, "key_two"); @@ -139,9 +138,9 @@ public void testUpdateReconOmDBWithNewSnapshot() throws Exception { reconOMMetadataManager, reconTaskController, reconUtilsMock, ozoneManagerProtocol); - Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()) + assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()) .get("/sampleVol/bucketOne/key_one")); - Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()) + assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()) .get("/sampleVol/bucketOne/key_two")); assertTrue(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot()); @@ -153,12 +152,13 @@ public void testUpdateReconOmDBWithNewSnapshot() throws Exception { } @Test - public void testReconOmDBCloseAndOpenNewSnapshotDb() throws Exception { + public void testReconOmDBCloseAndOpenNewSnapshotDb( + @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) + throws Exception { OMMetadataManager omMetadataManager = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata); ReconOMMetadataManager reconOMMetadataManager = - getTestReconOmMetadataManager(omMetadataManager, - temporaryFolder.newFolder()); + getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata); writeDataToOm(omMetadataManager, "key_one"); writeDataToOm(omMetadataManager, "key_two"); @@ -197,66 +197,58 @@ public void testReconOmDBCloseAndOpenNewSnapshotDb() throws Exception { } @Test - public void testGetOzoneManagerDBSnapshot() throws Exception { - - File reconOmSnapshotDbDir = temporaryFolder.newFolder(); + public void testGetOzoneManagerDBSnapshot(@TempDir File dirReconMetadata) + throws Exception { - File checkpointDir = Paths.get(reconOmSnapshotDbDir.getAbsolutePath(), + File checkpointDir = Paths.get(dirReconMetadata.getAbsolutePath(), "testGetOzoneManagerDBSnapshot").toFile(); checkpointDir.mkdir(); File file1 = Paths.get(checkpointDir.getAbsolutePath(), "file1") .toFile(); String str = "File1 Contents"; - BufferedWriter writer = null; - try { - writer = new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(file1), UTF_8)); - writer.write(str); - } finally { - if (writer != null) { - writer.close(); - } + + try (BufferedWriter writer1 = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(file1), UTF_8))) { + writer1.write(str); } File file2 = Paths.get(checkpointDir.getAbsolutePath(), "file2") .toFile(); str = "File2 Contents"; - try { - writer = new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(file2), UTF_8)); + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(file2), UTF_8))) { writer.write(str); - } finally { - writer.close(); } //Create test tar file. File tarFile = createTarFile(checkpointDir.toPath()); - InputStream fileInputStream = new FileInputStream(tarFile); - ReconUtils reconUtilsMock = getMockReconUtils(); - HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class); - when(httpURLConnectionMock.getInputStream()).thenReturn(fileInputStream); - when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean())) - .thenReturn(httpURLConnectionMock); - when(reconUtilsMock.getReconNodeDetails( - any(OzoneConfiguration.class))).thenReturn( - commonUtils.getReconNodeDetails()); - ReconOMMetadataManager reconOMMetadataManager = - mock(ReconOMMetadataManager.class); - ReconTaskController reconTaskController = getMockTaskController(); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - new OzoneManagerServiceProviderImpl(configuration, - reconOMMetadataManager, reconTaskController, reconUtilsMock, - ozoneManagerProtocol); - - DBCheckpoint checkpoint = ozoneManagerServiceProvider - .getOzoneManagerDBSnapshot(); - assertNotNull(checkpoint); - assertTrue(checkpoint.getCheckpointLocation().toFile().isDirectory()); - - File[] files = checkpoint.getCheckpointLocation().toFile().listFiles(); - assertNotNull(files); - assertEquals(2, files.length); + try (InputStream fileInputStream = new FileInputStream(tarFile)) { + ReconUtils reconUtilsMock = getMockReconUtils(); + HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class); + when(httpURLConnectionMock.getInputStream()).thenReturn(fileInputStream); + when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean())) + .thenReturn(httpURLConnectionMock); + when(reconUtilsMock.getReconNodeDetails( + any(OzoneConfiguration.class))).thenReturn( + commonUtils.getReconNodeDetails()); + ReconOMMetadataManager reconOMMetadataManager = + mock(ReconOMMetadataManager.class); + ReconTaskController reconTaskController = getMockTaskController(); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + new OzoneManagerServiceProviderImpl(configuration, + reconOMMetadataManager, reconTaskController, reconUtilsMock, + ozoneManagerProtocol); + + DBCheckpoint checkpoint = ozoneManagerServiceProvider + .getOzoneManagerDBSnapshot(); + assertNotNull(checkpoint); + assertTrue(checkpoint.getCheckpointLocation().toFile().isDirectory()); + + File[] files = checkpoint.getCheckpointLocation().toFile().listFiles(); + assertNotNull(files); + assertEquals(2, files.length); + } } @@ -265,12 +257,14 @@ static RocksDatabase getRocksDatabase(OMMetadataManager om) { } @Test - public void testGetAndApplyDeltaUpdatesFromOM() throws Exception { + public void testGetAndApplyDeltaUpdatesFromOM( + @TempDir File dirSrcOmMetadata, @TempDir File dirOmMetadata, + @TempDir File dirReconMetadata) throws Exception { // Writing 2 Keys into a source OM DB and collecting it in a // DBUpdatesWrapper. OMMetadataManager sourceOMMetadataMgr = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirSrcOmMetadata); writeDataToOm(sourceOMMetadataMgr, "key_one"); writeDataToOm(sourceOMMetadataMgr, "key_two"); @@ -288,12 +282,11 @@ public void testGetAndApplyDeltaUpdatesFromOM() throws Exception { // OM Service Provider's Metadata Manager. OMMetadataManager omMetadataManager = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata); OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, - getTestReconOmMetadataManager(omMetadataManager, - temporaryFolder.newFolder()), + getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClient(dbUpdatesWrapper)); @@ -326,12 +319,14 @@ public void testGetAndApplyDeltaUpdatesFromOM() throws Exception { } @Test - public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception { + public void testGetAndApplyDeltaUpdatesFromOMWithLimit( + @TempDir File dirSrcOmMetadata, @TempDir File dirOmMetadata, + @TempDir File dirReconMetadata) throws Exception { // Writing 2 Keys into a source OM DB and collecting it in a // DBUpdatesWrapper. OMMetadataManager sourceOMMetadataMgr = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirSrcOmMetadata); writeDataToOm(sourceOMMetadataMgr, "key_one"); writeDataToOm(sourceOMMetadataMgr, "key_two"); @@ -352,7 +347,7 @@ public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception { // OM Service Provider's Metadata Manager. OMMetadataManager omMetadataManager = - initializeNewOmMetadataManager(temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata); OzoneConfiguration withLimitConfiguration = new OzoneConfiguration(configuration); @@ -360,16 +355,15 @@ public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception { withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LOOP_LIMIT, 3); OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(withLimitConfiguration, - getTestReconOmMetadataManager(omMetadataManager, - temporaryFolder.newFolder()), + getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0], dbUpdatesWrapper[1], dbUpdatesWrapper[2], dbUpdatesWrapper[3])); - assertEquals(true, dbUpdatesWrapper[0].isDBUpdateSuccess()); - assertEquals(true, dbUpdatesWrapper[1].isDBUpdateSuccess()); - assertEquals(true, dbUpdatesWrapper[2].isDBUpdateSuccess()); - assertEquals(true, dbUpdatesWrapper[3].isDBUpdateSuccess()); + assertTrue(dbUpdatesWrapper[0].isDBUpdateSuccess()); + assertTrue(dbUpdatesWrapper[1].isDBUpdateSuccess()); + assertTrue(dbUpdatesWrapper[2].isDBUpdateSuccess()); + assertTrue(dbUpdatesWrapper[3].isDBUpdateSuccess()); OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager); @@ -400,12 +394,13 @@ public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception { } @Test - public void testSyncDataFromOMFullSnapshot() throws Exception { + public void testSyncDataFromOMFullSnapshot( + @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) + throws Exception { // Empty OM DB to start with. ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager( - initializeEmptyOmMetadataManager(temporaryFolder.newFolder()), - temporaryFolder.newFolder()); + initializeEmptyOmMetadataManager(dirOmMetadata), dirReconMetadata); ReconTaskStatusDao reconTaskStatusDaoMock = mock(ReconTaskStatusDao.class); doNothing().when(reconTaskStatusDaoMock) @@ -438,12 +433,13 @@ public void testSyncDataFromOMFullSnapshot() throws Exception { } @Test - public void testSyncDataFromOMDeltaUpdates() throws Exception { + public void testSyncDataFromOMDeltaUpdates( + @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) + throws Exception { // Non-Empty OM DB to start with. ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager( - initializeNewOmMetadataManager(temporaryFolder.newFolder()), - temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata), dirReconMetadata); ReconTaskStatusDao reconTaskStatusDaoMock = mock(ReconTaskStatusDao.class); doNothing().when(reconTaskStatusDaoMock) @@ -478,12 +474,13 @@ public void testSyncDataFromOMDeltaUpdates() throws Exception { } @Test - public void testSyncDataFromOMFullSnapshotForSNNFE() throws Exception { + public void testSyncDataFromOMFullSnapshotForSNNFE( + @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) + throws Exception { // Non-Empty OM DB to start with. ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager( - initializeNewOmMetadataManager(temporaryFolder.newFolder()), - temporaryFolder.newFolder()); + initializeNewOmMetadataManager(dirOmMetadata), dirReconMetadata); ReconTaskStatusDao reconTaskStatusDaoMock = mock(ReconTaskStatusDao.class); doNothing().when(reconTaskStatusDaoMock) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java index d75e99692c2..e5aa029a920 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.File; import java.io.IOException; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java index 96f58fdc416..6992c3100fb 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java @@ -39,7 +39,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.Assert; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -52,6 +51,9 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Test for NSSummaryTask. Create one bucket of each layout @@ -122,7 +124,7 @@ public static void setUp(@TempDir File tmpDir) throws Exception { NSSummary nonExistentSummary = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); - Assert.assertNull(nonExistentSummary); + assertNull(nonExistentSummary); populateOMDB(); @@ -151,10 +153,10 @@ public void setUp() throws Exception { reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); // Verify commit - Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); nSSummaryTask.reprocess(reconOMMetadataManager); - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); @@ -162,44 +164,44 @@ public void setUp() throws Exception { reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); nsSummaryForBucket3 = reconNamespaceSummaryManager.getNSSummary(BUCKET_THREE_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertNotNull(nsSummaryForBucket2); - Assert.assertNull(nsSummaryForBucket3); + assertNotNull(nsSummaryForBucket1); + assertNotNull(nsSummaryForBucket2); + assertNull(nsSummaryForBucket3); } @Test public void testReprocessNSSummaryNull() throws IOException { - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); } @Test public void testReprocessGetFiles() { - Assert.assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); - Assert.assertEquals(1, nsSummaryForBucket2.getNumOfFiles()); + assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); + assertEquals(1, nsSummaryForBucket2.getNumOfFiles()); - Assert.assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); - Assert.assertEquals(KEY_TWO_SIZE, nsSummaryForBucket2.getSizeOfFiles()); + assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); + assertEquals(KEY_TWO_SIZE, nsSummaryForBucket2.getSizeOfFiles()); } @Test public void testReprocessFileBucketSize() { int[] fileDistBucket1 = nsSummaryForBucket1.getFileSizeBucket(); int[] fileDistBucket2 = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket1.length); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket2.length); - Assert.assertEquals(1, fileDistBucket1[0]); + assertEquals(1, fileDistBucket1[0]); for (int i = 1; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - Assert.assertEquals(0, fileDistBucket1[i]); + assertEquals(0, fileDistBucket1[i]); } - Assert.assertEquals(1, fileDistBucket2[1]); + assertEquals(1, fileDistBucket2[1]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { if (i == 1) { continue; } - Assert.assertEquals(0, fileDistBucket2[i]); + assertEquals(0, fileDistBucket2[i]); } } @@ -225,13 +227,13 @@ public void setUp() throws IOException { nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket1); + assertNotNull(nsSummaryForBucket1); nsSummaryForBucket2 = reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket2); + assertNotNull(nsSummaryForBucket2); nsSummaryForBucket3 = reconNamespaceSummaryManager.getNSSummary(BUCKET_THREE_OBJECT_ID); - Assert.assertNull(nsSummaryForBucket3); + assertNull(nsSummaryForBucket3); } private OMUpdateEventBatch processEventBatch() throws IOException { @@ -271,31 +273,31 @@ private OMUpdateEventBatch processEventBatch() throws IOException { @Test public void testProcessUpdateFileSize() throws IOException { // file 1 is gone, so bucket 1 is empty now - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + assertNotNull(nsSummaryForBucket1); + assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); - Assert.assertEquals(0, childDirBucket1.size()); + assertEquals(0, childDirBucket1.size()); } @Test public void testProcessBucket() throws IOException { // file 5 is added under bucket 2, so bucket 2 has 2 keys now - Assert.assertNotNull(nsSummaryForBucket2); - Assert.assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); + assertNotNull(nsSummaryForBucket2); + assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); // key 2 + key 5 - Assert.assertEquals(KEY_TWO_SIZE + KEY_FIVE_SIZE, + assertEquals(KEY_TWO_SIZE + KEY_FIVE_SIZE, nsSummaryForBucket2.getSizeOfFiles()); int[] fileSizeDist = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileSizeDist.length); // 1025L - Assert.assertEquals(1, fileSizeDist[0]); + assertEquals(1, fileSizeDist[0]); // 2050L - Assert.assertEquals(1, fileSizeDist[1]); + assertEquals(1, fileSizeDist[1]); for (int i = 2; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - Assert.assertEquals(0, fileSizeDist[i]); + assertEquals(0, fileSizeDist[i]); } } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java index 771d22a121f..66c522cb4d7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java @@ -36,7 +36,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.Assert; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -52,6 +51,9 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Test for NSSummaryTaskWithFSO. @@ -138,7 +140,7 @@ public static void setUp(@TempDir File tmpDir) throws Exception { NSSummary nonExistentSummary = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); - Assert.assertNull(nonExistentSummary); + assertNull(nonExistentSummary); populateOMDB(); @@ -167,34 +169,34 @@ public void setUp() throws IOException { reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); // Verify commit - Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); // reinit Recon RocksDB's namespace CF. reconNamespaceSummaryManager.clearNSSummaryTable(); nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); nsSummaryForBucket2 = reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertNotNull(nsSummaryForBucket2); + assertNotNull(nsSummaryForBucket1); + assertNotNull(nsSummaryForBucket2); } @Test public void testReprocessNSSummaryNull() throws IOException { - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); } @Test public void testReprocessGetFiles() { - Assert.assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); - Assert.assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); + assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); + assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); - Assert.assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); - Assert.assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE, + assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); + assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE, nsSummaryForBucket2.getSizeOfFiles()); } @@ -202,22 +204,22 @@ public void testReprocessGetFiles() { public void testReprocessFileBucketSize() { int[] fileDistBucket1 = nsSummaryForBucket1.getFileSizeBucket(); int[] fileDistBucket2 = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket1.length); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket2.length); - Assert.assertEquals(1, fileDistBucket1[0]); + assertEquals(1, fileDistBucket1[0]); for (int i = 1; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - Assert.assertEquals(0, fileDistBucket1[i]); + assertEquals(0, fileDistBucket1[i]); } - Assert.assertEquals(1, fileDistBucket2[1]); - Assert.assertEquals(1, fileDistBucket2[2]); + assertEquals(1, fileDistBucket2[1]); + assertEquals(1, fileDistBucket2[2]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { if (i == 1 || i == 2) { continue; } - Assert.assertEquals(0, fileDistBucket2[i]); + assertEquals(0, fileDistBucket2[i]); } } @@ -226,11 +228,11 @@ public void testReprocessBucketDirs() { // Bucket one has one dir, bucket two has none. Set childDirBucketOne = nsSummaryForBucket1.getChildDir(); Set childDirBucketTwo = nsSummaryForBucket2.getChildDir(); - Assert.assertEquals(1, childDirBucketOne.size()); + assertEquals(1, childDirBucketOne.size()); bucketOneAns.clear(); bucketOneAns.add(DIR_ONE_OBJECT_ID); - Assert.assertEquals(bucketOneAns, childDirBucketOne); - Assert.assertEquals(0, childDirBucketTwo.size()); + assertEquals(bucketOneAns, childDirBucketOne); + assertEquals(0, childDirBucketTwo.size()); } @Test @@ -239,34 +241,34 @@ public void testReprocessDirsUnderDir() throws Exception { // Dir 1 has two dir: dir2 and dir3. NSSummary nsSummaryInDir1 = reconNamespaceSummaryManager .getNSSummary(DIR_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryInDir1); + assertNotNull(nsSummaryInDir1); Set childDirForDirOne = nsSummaryInDir1.getChildDir(); - Assert.assertEquals(2, childDirForDirOne.size()); + assertEquals(2, childDirForDirOne.size()); dirOneAns.clear(); dirOneAns.add(DIR_TWO_OBJECT_ID); dirOneAns.add(DIR_THREE_OBJECT_ID); - Assert.assertEquals(dirOneAns, childDirForDirOne); + assertEquals(dirOneAns, childDirForDirOne); NSSummary nsSummaryInDir2 = reconNamespaceSummaryManager .getNSSummary(DIR_TWO_OBJECT_ID); - Assert.assertEquals(1, nsSummaryInDir2.getNumOfFiles()); - Assert.assertEquals(KEY_THREE_SIZE, nsSummaryInDir2.getSizeOfFiles()); + assertEquals(1, nsSummaryInDir2.getNumOfFiles()); + assertEquals(KEY_THREE_SIZE, nsSummaryInDir2.getSizeOfFiles()); int[] fileDistForDir2 = nsSummaryInDir2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistForDir2.length); - Assert.assertEquals(1, fileDistForDir2[fileDistForDir2.length - 1]); + assertEquals(1, fileDistForDir2[fileDistForDir2.length - 1]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS - 1; ++i) { - Assert.assertEquals(0, fileDistForDir2[i]); + assertEquals(0, fileDistForDir2[i]); } - Assert.assertEquals(0, nsSummaryInDir2.getChildDir().size()); + assertEquals(0, nsSummaryInDir2.getChildDir().size()); // bucket should have empty dirName - Assert.assertEquals(0, nsSummaryForBucket1.getDirName().length()); - Assert.assertEquals(0, nsSummaryForBucket2.getDirName().length()); + assertEquals(0, nsSummaryForBucket1.getDirName().length()); + assertEquals(0, nsSummaryForBucket2.getDirName().length()); // check dirName is correctly written - Assert.assertEquals(DIR_ONE, nsSummaryInDir1.getDirName()); - Assert.assertEquals(DIR_TWO, nsSummaryInDir2.getDirName()); + assertEquals(DIR_ONE, nsSummaryInDir1.getDirName()); + assertEquals(DIR_TWO, nsSummaryInDir2.getDirName()); } } @@ -399,16 +401,16 @@ public void testProcessUpdateFileSize() throws IOException { NSSummary nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); // file 1 is gone, so bucket 1 is empty now - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + assertNotNull(nsSummaryForBucket1); + assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); // after put dir4, bucket1 now has two child dirs: dir1 and dir4 - Assert.assertEquals(2, childDirBucket1.size()); + assertEquals(2, childDirBucket1.size()); bucketOneAns.clear(); bucketOneAns.add(DIR_ONE_OBJECT_ID); bucketOneAns.add(DIR_FOUR_OBJECT_ID); - Assert.assertEquals(bucketOneAns, childDirBucket1); + assertEquals(bucketOneAns, childDirBucket1); } @Test @@ -418,31 +420,31 @@ public void testProcessBucket() throws IOException { // file 5 is added under bucket 2, so bucket 2 has 3 keys now // file 2 is updated with new datasize, // so file size dist for bucket 2 should be updated - Assert.assertNotNull(nsSummaryForBucket2); - Assert.assertEquals(3, nsSummaryForBucket2.getNumOfFiles()); + assertNotNull(nsSummaryForBucket2); + assertEquals(3, nsSummaryForBucket2.getNumOfFiles()); // key 4 + key 5 + updated key 2 - Assert.assertEquals(KEY_FOUR_SIZE + KEY_FIVE_SIZE + assertEquals(KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_TWO_UPDATE_SIZE, nsSummaryForBucket2.getSizeOfFiles()); int[] fileSizeDist = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileSizeDist.length); // 1023L and 100L - Assert.assertEquals(2, fileSizeDist[0]); + assertEquals(2, fileSizeDist[0]); // 2050L - Assert.assertEquals(1, fileSizeDist[2]); + assertEquals(1, fileSizeDist[2]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { if (i == 0 || i == 2) { continue; } - Assert.assertEquals(0, fileSizeDist[i]); + assertEquals(0, fileSizeDist[i]); } // after put dir5, bucket 2 now has one dir Set childDirBucket2 = nsSummaryForBucket2.getChildDir(); - Assert.assertEquals(1, childDirBucket2.size()); + assertEquals(1, childDirBucket2.size()); bucketTwoAns.add(DIR_FIVE_OBJECT_ID); - Assert.assertEquals(bucketTwoAns, childDirBucket2); + assertEquals(bucketTwoAns, childDirBucket2); } @Test @@ -450,15 +452,15 @@ public void testProcessDirDeleteRename() throws IOException { // after delete dir 3, dir 1 now has only one dir: dir2 NSSummary nsSummaryForDir1 = reconNamespaceSummaryManager .getNSSummary(DIR_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryForDir1); + assertNotNull(nsSummaryForDir1); Set childDirForDir1 = nsSummaryForDir1.getChildDir(); - Assert.assertEquals(1, childDirForDir1.size()); + assertEquals(1, childDirForDir1.size()); dirOneAns.clear(); dirOneAns.add(DIR_TWO_OBJECT_ID); - Assert.assertEquals(dirOneAns, childDirForDir1); + assertEquals(dirOneAns, childDirForDir1); // after renaming dir1, check its new name - Assert.assertEquals(DIR_ONE_RENAME, nsSummaryForDir1.getDirName()); + assertEquals(DIR_ONE_RENAME, nsSummaryForDir1.getDirName()); } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java index ca7a9d86b10..5ffd03cbb88 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java @@ -39,7 +39,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.Assert; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -54,6 +53,9 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Test for NSSummaryTaskWithLegacy. @@ -139,7 +141,7 @@ public static void setUp(@TempDir File tmpDir) throws Exception { NSSummary nonExistentSummary = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); - Assert.assertNull(nonExistentSummary); + assertNull(nonExistentSummary); populateOMDB(); @@ -168,34 +170,34 @@ public void setUp() throws IOException { reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); // Verify commit - Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); // reinit Recon RocksDB's namespace CF. reconNamespaceSummaryManager.clearNSSummaryTable(); nSSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); nsSummaryForBucket2 = reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertNotNull(nsSummaryForBucket2); + assertNotNull(nsSummaryForBucket1); + assertNotNull(nsSummaryForBucket2); } @Test public void testReprocessNSSummaryNull() throws IOException { - Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); } @Test public void testReprocessGetFiles() { - Assert.assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); - Assert.assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); + assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); + assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); - Assert.assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); - Assert.assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE, + assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); + assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE, nsSummaryForBucket2.getSizeOfFiles()); } @@ -203,22 +205,22 @@ public void testReprocessGetFiles() { public void testReprocessFileBucketSize() { int[] fileDistBucket1 = nsSummaryForBucket1.getFileSizeBucket(); int[] fileDistBucket2 = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket1.length); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistBucket2.length); - Assert.assertEquals(1, fileDistBucket1[0]); + assertEquals(1, fileDistBucket1[0]); for (int i = 1; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - Assert.assertEquals(0, fileDistBucket1[i]); + assertEquals(0, fileDistBucket1[i]); } - Assert.assertEquals(1, fileDistBucket2[1]); - Assert.assertEquals(1, fileDistBucket2[2]); + assertEquals(1, fileDistBucket2[1]); + assertEquals(1, fileDistBucket2[2]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { if (i == 1 || i == 2) { continue; } - Assert.assertEquals(0, fileDistBucket2[i]); + assertEquals(0, fileDistBucket2[i]); } } @@ -227,11 +229,11 @@ public void testReprocessBucketDirs() { // Bucket one has one dir, bucket two has none. Set childDirBucketOne = nsSummaryForBucket1.getChildDir(); Set childDirBucketTwo = nsSummaryForBucket2.getChildDir(); - Assert.assertEquals(1, childDirBucketOne.size()); + assertEquals(1, childDirBucketOne.size()); bucketOneAns.clear(); bucketOneAns.add(DIR_ONE_OBJECT_ID); - Assert.assertEquals(bucketOneAns, childDirBucketOne); - Assert.assertEquals(0, childDirBucketTwo.size()); + assertEquals(bucketOneAns, childDirBucketOne); + assertEquals(0, childDirBucketTwo.size()); } @Test @@ -240,35 +242,35 @@ public void testReprocessDirsUnderDir() throws Exception { // Dir 1 has two dir: dir2 and dir3. NSSummary nsSummaryInDir1 = reconNamespaceSummaryManager .getNSSummary(DIR_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryInDir1); + assertNotNull(nsSummaryInDir1); Set childDirForDirOne = nsSummaryInDir1.getChildDir(); - Assert.assertEquals(2, childDirForDirOne.size()); + assertEquals(2, childDirForDirOne.size()); dirOneAns.clear(); dirOneAns.add(DIR_TWO_OBJECT_ID); dirOneAns.add(DIR_THREE_OBJECT_ID); - Assert.assertEquals(dirOneAns, childDirForDirOne); + assertEquals(dirOneAns, childDirForDirOne); NSSummary nsSummaryInDir2 = reconNamespaceSummaryManager .getNSSummary(DIR_TWO_OBJECT_ID); - Assert.assertEquals(1, nsSummaryInDir2.getNumOfFiles()); - Assert.assertEquals(KEY_THREE_SIZE, nsSummaryInDir2.getSizeOfFiles()); + assertEquals(1, nsSummaryInDir2.getNumOfFiles()); + assertEquals(KEY_THREE_SIZE, nsSummaryInDir2.getSizeOfFiles()); int[] fileDistForDir2 = nsSummaryInDir2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileDistForDir2.length); - Assert.assertEquals(1, + assertEquals(1, fileDistForDir2[fileDistForDir2.length - 1]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS - 1; ++i) { - Assert.assertEquals(0, fileDistForDir2[i]); + assertEquals(0, fileDistForDir2[i]); } - Assert.assertEquals(0, nsSummaryInDir2.getChildDir().size()); + assertEquals(0, nsSummaryInDir2.getChildDir().size()); // bucket should have empty dirName - Assert.assertEquals(0, nsSummaryForBucket1.getDirName().length()); - Assert.assertEquals(0, nsSummaryForBucket2.getDirName().length()); + assertEquals(0, nsSummaryForBucket1.getDirName().length()); + assertEquals(0, nsSummaryForBucket2.getDirName().length()); // check dirName is correctly written - Assert.assertEquals(DIR_ONE, nsSummaryInDir1.getDirName()); - Assert.assertEquals(DIR_ONE + OM_KEY_PREFIX + DIR_TWO, + assertEquals(DIR_ONE, nsSummaryInDir1.getDirName()); + assertEquals(DIR_ONE + OM_KEY_PREFIX + DIR_TWO, nsSummaryInDir2.getDirName()); } } @@ -297,10 +299,10 @@ public void setUp() throws IOException { nsSummaryForBucket1 = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket1); + assertNotNull(nsSummaryForBucket1); nsSummaryForBucket2 = reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); - Assert.assertNotNull(nsSummaryForBucket2); + assertNotNull(nsSummaryForBucket2); } private OMUpdateEventBatch processEventBatch() throws IOException { @@ -436,16 +438,16 @@ private OMUpdateEventBatch processEventBatch() throws IOException { @Test public void testProcessUpdateFileSize() throws IOException { // file 1 is gone, so bucket 1 is empty now - Assert.assertNotNull(nsSummaryForBucket1); - Assert.assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + assertNotNull(nsSummaryForBucket1); + assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); // after put dir4, bucket1 now has two child dirs: dir1 and dir4 - Assert.assertEquals(2, childDirBucket1.size()); + assertEquals(2, childDirBucket1.size()); bucketOneAns.clear(); bucketOneAns.add(DIR_ONE_OBJECT_ID); bucketOneAns.add(DIR_FOUR_OBJECT_ID); - Assert.assertEquals(bucketOneAns, childDirBucket1); + assertEquals(bucketOneAns, childDirBucket1); } @Test @@ -453,31 +455,31 @@ public void testProcessBucket() throws IOException { // file 5 is added under bucket 2, so bucket 2 has 3 keys now // file 2 is updated with new datasize, // so file size dist for bucket 2 should be updated - Assert.assertNotNull(nsSummaryForBucket2); - Assert.assertEquals(3, nsSummaryForBucket2.getNumOfFiles()); + assertNotNull(nsSummaryForBucket2); + assertEquals(3, nsSummaryForBucket2.getNumOfFiles()); // key 4 + key 5 + updated key 2 - Assert.assertEquals(KEY_FOUR_SIZE + KEY_FIVE_SIZE + assertEquals(KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_TWO_UPDATE_SIZE, nsSummaryForBucket2.getSizeOfFiles()); int[] fileSizeDist = nsSummaryForBucket2.getFileSizeBucket(); - Assert.assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, + assertEquals(ReconConstants.NUM_OF_FILE_SIZE_BINS, fileSizeDist.length); // 1023L and 100L - Assert.assertEquals(2, fileSizeDist[0]); + assertEquals(2, fileSizeDist[0]); // 2050L - Assert.assertEquals(1, fileSizeDist[2]); + assertEquals(1, fileSizeDist[2]); for (int i = 0; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { if (i == 0 || i == 2) { continue; } - Assert.assertEquals(0, fileSizeDist[i]); + assertEquals(0, fileSizeDist[i]); } // after put dir5, bucket 2 now has one dir Set childDirBucket2 = nsSummaryForBucket2.getChildDir(); - Assert.assertEquals(1, childDirBucket2.size()); + assertEquals(1, childDirBucket2.size()); bucketTwoAns.add(DIR_FIVE_OBJECT_ID); - Assert.assertEquals(bucketTwoAns, childDirBucket2); + assertEquals(bucketTwoAns, childDirBucket2); } @Test @@ -485,15 +487,15 @@ public void testProcessDirDeleteRename() throws IOException { // after delete dir 3, dir 1 now has only one dir: dir2 NSSummary nsSummaryForDir1 = reconNamespaceSummaryManager .getNSSummary(DIR_ONE_OBJECT_ID); - Assert.assertNotNull(nsSummaryForDir1); + assertNotNull(nsSummaryForDir1); Set childDirForDir1 = nsSummaryForDir1.getChildDir(); - Assert.assertEquals(1, childDirForDir1.size()); + assertEquals(1, childDirForDir1.size()); dirOneAns.clear(); dirOneAns.add(DIR_TWO_OBJECT_ID); - Assert.assertEquals(dirOneAns, childDirForDir1); + assertEquals(dirOneAns, childDirForDir1); // after renaming dir1, check its new name - Assert.assertEquals(DIR_ONE_RENAME, nsSummaryForDir1.getDirName()); + assertEquals(DIR_ONE_RENAME, nsSummaryForDir1.getDirName()); } } From 8c97e1e5840882608f6bbb8a6963fb24ea885364 Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Mon, 27 Nov 2023 19:44:23 +0200 Subject: [PATCH 06/51] HDDS-7329. Extend ozone admin datanode usageinfo and list info to accept hostname parameter (#3835) Co-authored-by: Doroszlai, Attila --- .../hadoop/hdds/scm/client/ScmClient.java | 6 +- .../StorageContainerLocationProtocol.java | 6 +- ...ocationProtocolClientSideTranslatorPB.java | 6 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 97 ++++++++++--------- .../scm/server/SCMClientProtocolServer.java | 10 +- .../hdds/scm/node/TestSCMNodeManager.java | 59 ++++------- .../scm/cli/ContainerOperationClient.java | 4 +- .../scm/cli/datanode/ListInfoSubcommand.java | 9 ++ .../scm/cli/datanode/UsageInfoSubcommand.java | 20 ++-- .../main/smoketest/admincli/datanode.robot | 60 +++++++++--- 10 files changed, 150 insertions(+), 127 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 179bf489cb1..b03cead27e7 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -400,15 +400,15 @@ List getFailedDeletedBlockTxn(int count, int resetDeletedBlockRetryCount(List txIDs) throws IOException; /** - * Get usage information of datanode by ipaddress or uuid. + * Get usage information of datanode by address or uuid. * - * @param ipaddress datanode ipaddress String + * @param address datanode address String * @param uuid datanode uuid String * @return List of DatanodeUsageInfoProto. Each element contains info such as * capacity, SCMused, and remaining space. * @throws IOException */ - List getDatanodeUsageInfo(String ipaddress, + List getDatanodeUsageInfo(String address, String uuid) throws IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 7cdb7bf7d57..b587cc924b0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -410,9 +410,9 @@ StartContainerBalancerResponseProto startContainerBalancer( boolean getContainerBalancerStatus() throws IOException; /** - * Get Datanode usage information by ip or uuid. + * Get Datanode usage information by ip or hostname or uuid. * - * @param ipaddress datanode IP address String + * @param address datanode IP address or Hostname String * @param uuid datanode UUID String * @param clientVersion Client's version number * @return List of DatanodeUsageInfoProto. Each element contains info such as @@ -421,7 +421,7 @@ StartContainerBalancerResponseProto startContainerBalancer( * @see org.apache.hadoop.ozone.ClientVersion */ List getDatanodeUsageInfo( - String ipaddress, String uuid, int clientVersion) throws IOException; + String address, String uuid, int clientVersion) throws IOException; /** * Get usage information of most or least used datanodes. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 79ea9669331..80db9047baa 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -932,7 +932,7 @@ public boolean getContainerBalancerStatus() throws IOException { /** * Builds request for datanode usage information and receives response. * - * @param ipaddress Address String + * @param address Address String * @param uuid UUID String * @return List of DatanodeUsageInfoProto. Each element contains info such as * capacity, SCMUsed, and remaining space. @@ -940,11 +940,11 @@ public boolean getContainerBalancerStatus() throws IOException { */ @Override public List getDatanodeUsageInfo( - String ipaddress, String uuid, int clientVersion) throws IOException { + String address, String uuid, int clientVersion) throws IOException { DatanodeUsageInfoRequestProto request = DatanodeUsageInfoRequestProto.newBuilder() - .setIpaddress(ipaddress) + .setIpaddress(address) .setUuid(uuid) .build(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 0d323b996c1..3103d5a7d4a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -82,6 +82,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -379,19 +380,15 @@ public RegisteredCommand register( datanodeDetails.setIpAddress(dnAddress.getHostAddress()); } - String dnsName; - String networkLocation; + final String ipAddress = datanodeDetails.getIpAddress(); + final String hostName = datanodeDetails.getHostName(); datanodeDetails.setNetworkName(datanodeDetails.getUuidString()); - if (useHostname) { - dnsName = datanodeDetails.getHostName(); - } else { - dnsName = datanodeDetails.getIpAddress(); - } - networkLocation = nodeResolve(dnsName); + String networkLocation = nodeResolve(useHostname ? hostName : ipAddress); if (networkLocation != null) { datanodeDetails.setNetworkLocation(networkLocation); } + final UUID uuid = datanodeDetails.getUuid(); if (!isNodeRegistered(datanodeDetails)) { try { clusterMap.add(datanodeDetails); @@ -399,14 +396,14 @@ public RegisteredCommand register( // Check that datanode in nodeStateManager has topology parent set DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); - addToDnsToUuidMap(dnsName, datanodeDetails.getUuid()); + addToDnsToUuidMap(uuid, ipAddress, hostName); // Updating Node Report, as registration is successful processNodeReport(datanodeDetails, nodeReport); - LOG.info("Registered Data node : {}", datanodeDetails.toDebugString()); + LOG.info("Registered datanode: {}", datanodeDetails.toDebugString()); scmNodeEventPublisher.fireEvent(SCMEvents.NEW_NODE, datanodeDetails); } catch (NodeAlreadyExistsException e) { if (LOG.isTraceEnabled()) { - LOG.trace("Datanode is already registered. Datanode: {}", + LOG.trace("Datanode is already registered: {}", datanodeDetails); } } catch (NodeNotFoundException e) { @@ -416,32 +413,20 @@ public RegisteredCommand register( } else { // Update datanode if it is registered but the ip or hostname changes try { - final DatanodeInfo datanodeInfo = - nodeStateManager.getNode(datanodeDetails); - if (!datanodeInfo.getIpAddress().equals(datanodeDetails.getIpAddress()) - || !datanodeInfo.getHostName() - .equals(datanodeDetails.getHostName())) { - LOG.info("Updating data node {} from {} to {}", + final DatanodeInfo oldNode = nodeStateManager.getNode(datanodeDetails); + if (updateDnsToUuidMap(oldNode.getHostName(), oldNode.getIpAddress(), + hostName, ipAddress, uuid)) { + LOG.info("Updating datanode {} from {} to {}", datanodeDetails.getUuidString(), - datanodeInfo, + oldNode, datanodeDetails); - clusterMap.update(datanodeInfo, datanodeDetails); - - String oldDnsName; - if (useHostname) { - oldDnsName = datanodeInfo.getHostName(); - } else { - oldDnsName = datanodeInfo.getIpAddress(); - } - updateDnsToUuidMap(oldDnsName, dnsName, datanodeDetails.getUuid()); - + clusterMap.update(oldNode, datanodeDetails); nodeStateManager.updateNode(datanodeDetails, layoutInfo); DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); processNodeReport(datanodeDetails, nodeReport); - LOG.info("Updated Datanode to: {}", dn); - scmNodeEventPublisher - .fireEvent(SCMEvents.NODE_ADDRESS_UPDATE, dn); + LOG.info("Updated datanode to: {}", dn); + scmNodeEventPublisher.fireEvent(SCMEvents.NODE_ADDRESS_UPDATE, dn); } } catch (NodeNotFoundException e) { LOG.error("Cannot find datanode {} from nodeStateManager", @@ -458,31 +443,49 @@ public RegisteredCommand register( /** * Add an entry to the dnsToUuidMap, which maps hostname / IP to the DNs * running on that host. As each address can have many DNs running on it, - * this is a one to many mapping. + * and each host can have multiple addresses, + * this is a many to many mapping. * - * @param addr the hostname or IP of the node * @param uuid the UUID of the registered node. + * @param addresses hostname and/or IP of the node */ - private synchronized void addToDnsToUuidMap(String addr, UUID uuid) { - dnsToUuidMap.computeIfAbsent(addr, k -> ConcurrentHashMap.newKeySet()) - .add(uuid); + private synchronized void addToDnsToUuidMap(UUID uuid, String... addresses) { + for (String addr : addresses) { + if (!Strings.isNullOrEmpty(addr)) { + dnsToUuidMap.computeIfAbsent(addr, k -> ConcurrentHashMap.newKeySet()) + .add(uuid); + } + } } - private synchronized void removeFromDnsToUuidMap(String addr, UUID uuid) { - Set dnSet = dnsToUuidMap.get(addr); - if (dnSet != null && dnSet.remove(uuid) && dnSet.isEmpty()) { - dnsToUuidMap.remove(addr); + private synchronized void removeFromDnsToUuidMap(UUID uuid, String address) { + if (address != null) { + Set dnSet = dnsToUuidMap.get(address); + if (dnSet != null && dnSet.remove(uuid) && dnSet.isEmpty()) { + dnsToUuidMap.remove(address); + } } } - private synchronized void updateDnsToUuidMap( - String oldDnsName, String newDnsName, UUID uuid) { - Preconditions.checkNotNull(oldDnsName, "old address == null"); - Preconditions.checkNotNull(newDnsName, "new address == null"); - if (!oldDnsName.equals(newDnsName)) { - removeFromDnsToUuidMap(oldDnsName, uuid); - addToDnsToUuidMap(newDnsName, uuid); + private boolean updateDnsToUuidMap( + String oldHostName, String oldIpAddress, + String newHostName, String newIpAddress, + UUID uuid) { + final boolean ipChanged = !Objects.equals(oldIpAddress, newIpAddress); + final boolean hostNameChanged = !Objects.equals(oldHostName, newHostName); + if (ipChanged || hostNameChanged) { + synchronized (this) { + if (ipChanged) { + removeFromDnsToUuidMap(uuid, oldIpAddress); + addToDnsToUuidMap(uuid, newIpAddress); + } + if (hostNameChanged) { + removeFromDnsToUuidMap(uuid, oldHostName); + addToDnsToUuidMap(uuid, newHostName); + } + } } + return ipChanged || hostNameChanged; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 21ab8a25016..315be62b876 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1106,9 +1106,9 @@ public boolean getContainerBalancerStatus() { /** * Get Datanode usage info such as capacity, SCMUsed, and remaining by ip - * or uuid. + * or hostname or uuid. * - * @param ipaddress Datanode Address String + * @param address Datanode Address String * @param uuid Datanode UUID String * @return List of DatanodeUsageInfoProto. Each element contains usage info * such as capacity, SCMUsed, and remaining space. @@ -1116,7 +1116,7 @@ public boolean getContainerBalancerStatus() { */ @Override public List getDatanodeUsageInfo( - String ipaddress, String uuid, int clientVersion) throws IOException { + String address, String uuid, int clientVersion) throws IOException { // check admin authorisation try { @@ -1130,8 +1130,8 @@ public List getDatanodeUsageInfo( List nodes = new ArrayList<>(); if (!Strings.isNullOrEmpty(uuid)) { nodes.add(scm.getScmNodeManager().getNodeByUuid(uuid)); - } else if (!Strings.isNullOrEmpty(ipaddress)) { - nodes = scm.getScmNodeManager().getNodesByAddress(ipaddress); + } else if (!Strings.isNullOrEmpty(address)) { + nodes = scm.getScmNodeManager().getNodesByAddress(address); } else { throw new IOException( "Could not get datanode with the specified parameters." diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index e75f6e6f41a..d13cd8952ae 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1794,26 +1794,6 @@ public void testHandlingSCMCommandEvent() } } - /** - * Test add node into network topology during node register. Datanode - * uses Ip address to resolve network location. - */ - @Test - public void testScmRegisterNodeWithIpAddress() - throws IOException, InterruptedException, AuthenticationException { - testScmRegisterNodeWithNetworkTopology(false); - } - - /** - * Test add node into network topology during node register. Datanode - * uses hostname to resolve network location. - */ - @Test - public void testScmRegisterNodeWithHostname() - throws IOException, InterruptedException, AuthenticationException { - testScmRegisterNodeWithNetworkTopology(true); - } - /** * Test add node into a 4-layer network topology during node register. */ @@ -1859,11 +1839,15 @@ public void testScmRegisterNodeWith4LayerNetworkTopology() } } - private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testScmRegisterNodeWithNetworkTopology(boolean useHostname) throws IOException, InterruptedException, AuthenticationException { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, MILLISECONDS); + conf.setBoolean(DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME, + useHostname); // create table mapping file String[] hostNames = {"host1", "host2", "host3", "host4"}; @@ -1875,9 +1859,7 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) conf.set(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, "org.apache.hadoop.net.TableMapping"); conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mapFile); - if (useHostname) { - conf.set(DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME, "true"); - } + final int nodeCount = hostNames.length; // use default IP address to resolve node try (SCMNodeManager nodeManager = createNodeManager(conf)) { @@ -1899,13 +1881,11 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) assertEquals("/rack1", node.getNetworkLocation())); // test get node - if (useHostname) { - Arrays.stream(hostNames).forEach(hostname -> assertNotEquals(0, - nodeManager.getNodesByAddress(hostname).size())); - } else { - Arrays.stream(ipAddress).forEach(ip -> assertNotEquals(0, - nodeManager.getNodesByAddress(ip).size())); - } + Arrays.stream(hostNames).forEach(hostname -> assertNotEquals(0, + nodeManager.getNodesByAddress(hostname).size())); + + Arrays.stream(ipAddress).forEach(ip -> assertNotEquals(0, + nodeManager.getNodesByAddress(ip).size())); } } @@ -1998,15 +1978,14 @@ void testGetNodesByAddress(boolean useHostname) } // test get node assertEquals(0, nodeManager.getNodesByAddress(null).size()); - if (useHostname) { - assertEquals(2, nodeManager.getNodesByAddress("host1").size()); - assertEquals(1, nodeManager.getNodesByAddress("host2").size()); - assertEquals(0, nodeManager.getNodesByAddress("unknown").size()); - } else { - assertEquals(2, nodeManager.getNodesByAddress("1.2.3.4").size()); - assertEquals(1, nodeManager.getNodesByAddress("2.3.4.5").size()); - assertEquals(0, nodeManager.getNodesByAddress("1.9.8.7").size()); - } + + assertEquals(2, nodeManager.getNodesByAddress("host1").size()); + assertEquals(1, nodeManager.getNodesByAddress("host2").size()); + assertEquals(0, nodeManager.getNodesByAddress("unknown").size()); + + assertEquals(2, nodeManager.getNodesByAddress("1.2.3.4").size()); + assertEquals(1, nodeManager.getNodesByAddress("2.3.4.5").size()); + assertEquals(0, nodeManager.getNodesByAddress("1.9.8.7").size()); } } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index ee33c6b0e76..13e8aa77af9 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -519,8 +519,8 @@ public int resetDeletedBlockRetryCount(List txIDs) throws IOException { @Override public List getDatanodeUsageInfo( - String ipaddress, String uuid) throws IOException { - return storageContainerLocationClient.getDatanodeUsageInfo(ipaddress, + String address, String uuid) throws IOException { + return storageContainerLocationClient.getDatanodeUsageInfo(address, uuid, ClientVersion.CURRENT_VERSION); } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 08daa1fa718..db12ee2aacb 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -52,6 +52,11 @@ public class ListInfoSubcommand extends ScmSubcommand { defaultValue = "") private String uuid; + @CommandLine.Option(names = {"--hostname"}, + description = "Show info by datanode hostname.", + defaultValue = "") + private String hostname; + @CommandLine.Option(names = {"--operational-state"}, description = "Show info by datanode NodeOperationalState(" + "IN_SERVICE, " + @@ -82,6 +87,10 @@ public void execute(ScmClient scmClient) throws IOException { allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress() .compareToIgnoreCase(ipaddress) == 0); } + if (!Strings.isNullOrEmpty(hostname)) { + allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName() + .compareToIgnoreCase(hostname) == 0); + } if (!Strings.isNullOrEmpty(uuid)) { allNodes = allNodes.filter(p -> p.getDatanodeDetails().getUuidString().equals(uuid)); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java index 5bd6b42b449..d46513b24bb 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java @@ -45,7 +45,7 @@ name = "usageinfo", description = "List usage information " + "(such as Capacity, SCMUsed, Remaining) of a datanode by IP address " + - "or UUID", + "or Host name or UUID", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) public class UsageInfoSubcommand extends ScmSubcommand { @@ -61,9 +61,10 @@ public class UsageInfoSubcommand extends ScmSubcommand { private ExclusiveArguments exclusiveArguments; private static class ExclusiveArguments { - @CommandLine.Option(names = {"--ip"}, paramLabel = "IP", description = - "Show info by datanode ip address.", defaultValue = "") - private String ipaddress; + @CommandLine.Option(names = {"--address"}, paramLabel = "ADDRESS", + description = "Show info by datanode ip or hostname address.", + defaultValue = "") + private String address; @CommandLine.Option(names = {"--uuid"}, paramLabel = "UUID", description = "Show info by datanode UUID.", defaultValue = "") @@ -98,10 +99,10 @@ public void execute(ScmClient scmClient) throws IOException { throw new IOException("Count must be an integer greater than 0."); } - // fetch info by ip or uuid - if (!Strings.isNullOrEmpty(exclusiveArguments.ipaddress) || + // fetch info by ip or hostname or uuid + if (!Strings.isNullOrEmpty(exclusiveArguments.address) || !Strings.isNullOrEmpty(exclusiveArguments.uuid)) { - infoList = scmClient.getDatanodeUsageInfo(exclusiveArguments.ipaddress, + infoList = scmClient.getDatanodeUsageInfo(exclusiveArguments.address, exclusiveArguments.uuid); } else { // get info of most used or least used nodes infoList = scmClient.getDatanodeUsageInfo(exclusiveArguments.mostUsed, @@ -129,8 +130,9 @@ public void execute(ScmClient scmClient) throws IOException { private void printInfo(DatanodeUsage info) { System.out.printf("%-13s: %s %n", "UUID", info.getDatanodeDetails().getUuid()); - System.out.printf("%-13s: %s (%s) %n", "IP Address", - info.getDatanodeDetails().getIpAddress(), + System.out.printf("%-13s: %s %n", "IP Address", + info.getDatanodeDetails().getIpAddress()); + System.out.printf("%-13s: %s %n", "Hostname", info.getDatanodeDetails().getHostName()); // print capacity in a readable format System.out.printf("%-13s: %s (%s) %n", "Capacity", info.getCapacity() diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index 340d3de8257..b4ee5b95290 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -17,8 +17,18 @@ Documentation Test ozone admin datanode command Library BuiltIn Resource ../commonlib.robot +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 5 minutes +*** Keywords *** +Assert Output + [arguments] ${output} ${expected} ${uuid} + Should contain ${output} Datanode: ${uuid} + ${datanodes} = Get Lines Containing String ${output} Datanode: + @{lines} = Split To Lines ${datanodes} + ${count} = Get Length ${lines} + Should Be Equal As Integers ${count} ${expected} + *** Test Cases *** List datanodes Execute ozone admin datanode list > datanode.list @@ -29,31 +39,51 @@ List datanodes Filter list by UUID ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' ${output} = Execute ozone admin datanode list --id "${uuid}" - Should contain ${output} Datanode: ${uuid} - ${datanodes} = Get Lines Containing String ${output} Datanode: - @{lines} = Split To Lines ${datanodes} - ${count} = Get Length ${lines} - Should Be Equal As Integers ${count} 1 + Assert Output ${output} 1 ${uuid} + +Filter list by Ip address + ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' + ${ip} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$3 }' | awk -F '[/]' '{ print \$3 }' + ${output} = Execute ozone admin datanode list --ip "${ip}" + Assert Output ${output} 1 ${uuid} + +Filter list by Hostname + ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' + ${hostname} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$3 }' | awk -F '[/]' '{ print \$4 }' + ${output} = Execute ozone admin datanode list --hostname "${hostname}" + Assert Output ${output} 1 ${uuid} Filter list by NodeOperationalState ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' ${expected} = Execute grep -c 'Operational State: IN_SERVICE' datanode.list ${output} = Execute ozone admin datanode list --operational-state IN_SERVICE - Should contain ${output} Datanode: ${uuid} - ${datanodes} = Get Lines Containing String ${output} Datanode: - @{lines} = Split To Lines ${datanodes} - ${count} = Get Length ${lines} - Should Be Equal As Integers ${count} ${expected} + Assert Output ${output} ${expected} ${uuid} Filter list by NodeState ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' ${expected} = Execute grep -c 'Health State: HEALTHY' datanode.list ${output} = Execute ozone admin datanode list --node-state HEALTHY - Should contain ${output} Datanode: ${uuid} - ${datanodes} = Get Lines Containing String ${output} Datanode: - @{lines} = Split To Lines ${datanodes} - ${count} = Get Length ${lines} - Should Be Equal As Integers ${count} ${expected} + Assert Output ${output} ${expected} ${uuid} + +Get usage info by UUID + ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' + ${output} = Execute ozone admin datanode usageinfo --uuid "${uuid}" + Should contain ${output} Usage Information (1 Datanodes) + +Get usage info by Ip address + ${ip} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$3 }' | awk -F '[/]' '{ print \$3 }' + ${output} = Execute ozone admin datanode usageinfo --address "${ip}" + Should contain ${output} Usage Information (1 Datanodes) + +Get usage info by Hostname + ${hostname} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$3 }' | awk -F '[/]' '{ print \$4 }' + ${output} = Execute ozone admin datanode usageinfo --address "${hostname}" + Should contain ${output} Usage Information (1 Datanodes) + +Get usage info with invalid address + ${uuid} = Execute grep '^Datanode:' datanode.list | head -1 | awk '{ print \$2 }' + ${output} = Execute ozone admin datanode usageinfo --address "${uuid}" + Should contain ${output} Usage Information (0 Datanodes) Incomplete command ${output} = Execute And Ignore Error ozone admin datanode From fbe584c0cc6e33b737b9b6cf31ee592018d8e594 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Mon, 27 Nov 2023 12:25:07 -0800 Subject: [PATCH 07/51] HDDS-8040. Increase ratis log segment default size in datanode. (#5671) --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 ++++++-- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 ++++ .../src/main/resources/ozone-default.xml | 20 +++++++++++++------ .../server/ratis/XceiverServerRatis.java | 10 +++++++++- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 2d8f8c06c85..1deff3e409d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -66,11 +66,15 @@ public final class ScmConfigKeys { public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY = "dfs.container.ratis.segment.size"; public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT = - "1MB"; + "64MB"; + public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY = + "dfs.container.ratis.segment.buffer.size"; + public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT = + "2MB"; public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY = "dfs.container.ratis.segment.preallocated.size"; public static final String - DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "16KB"; + DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "4MB"; public static final String DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT = "dfs.container.ratis.statemachinedata.sync.timeout"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 0b62b887a3c..338fd778b09 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -352,6 +352,10 @@ public final class OzoneConfigKeys { = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY; public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT; + public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY + = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY; + public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT + = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT; public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY; public static final String diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c84aa911501..257fcbb77ea 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -314,18 +314,26 @@ dfs.container.ratis.segment.size - 1MB + 64MB OZONE, RATIS, PERFORMANCE - The size of the raft segment used by Apache Ratis on datanodes. - (1 MB by default) + The size of the raft segment file used + by Apache Ratis on datanodes. (64 MB by default) + + + + dfs.container.ratis.segment.buffer.size + 2MB + OZONE, RATIS, PERFORMANCE + The size of the raft segment buffer used + by Apache Ratis on datanodes. (2 MB by default) dfs.container.ratis.segment.preallocated.size - 16KB + 4MB OZONE, RATIS, PERFORMANCE - The size of the buffer which is preallocated for raft segment - used by Apache Ratis on datanodes.(16 KB by default) + The pre-allocated file size for raft segment used + by Apache Ratis on datanodes. (4 MB by default) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index eb5536faadf..2a57a32b39c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -103,6 +103,7 @@ import org.apache.ratis.server.RaftServerRpc; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.storage.RaftStorage; +import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.TraditionalBinaryPrefix; @@ -455,8 +456,15 @@ private void setRaftSegmentAndWriteBufferSize(RaftProperties properties) { StorageUnit.BYTES); RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf(raftSegmentSize)); + final long raftSegmentBufferSize = (long) conf.getStorageSize( + OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY, + OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT, + StorageUnit.BYTES); RaftServerConfigKeys.Log.setWriteBufferSize(properties, - SizeInBytes.valueOf(raftSegmentSize)); + SizeInBytes.valueOf(raftSegmentBufferSize)); + Preconditions.assertTrue(raftSegmentBufferSize <= raftSegmentSize, + () -> "raftSegmentBufferSize = " + raftSegmentBufferSize + + " > raftSegmentSize = " + raftSegmentSize); } private RpcType setRpcType(RaftProperties properties) { From 7da62e6d275764fdaf6a580ded9ac16d0e26ff86 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Tue, 28 Nov 2023 07:19:06 +0530 Subject: [PATCH 08/51] HDDS-3438. Fix and enable TestContainerServer#testClientServerWithContainerDispatcher (#5667) --- .../container/server/TestContainerServer.java | 103 ++++++++---------- 1 file changed, 46 insertions(+), 57 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index ccc62ddeab6..7e5db1f8e57 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -20,11 +20,13 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -56,30 +58,31 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; import org.apache.ozone.test.GenericTestUtils; import com.google.common.collect.Maps; + +import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.rpc.RpcType; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.ratis.rpc.SupportedRpcType.GRPC; import org.apache.ratis.util.function.CheckedBiConsumer; import org.apache.ratis.util.function.CheckedBiFunction; import org.junit.Assert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.apache.ozone.test.tag.Slow; import org.junit.jupiter.api.Test; -import static org.mockito.Mockito.mock; /** * Test Containers. */ -@Slow public class TestContainerServer { static final String TEST_DIR = GenericTestUtils.getTestDir("dfs") .getAbsolutePath() + File.separator; @@ -182,61 +185,47 @@ static void runTestClientServer( } } - @Unhealthy("Fails with StatusRuntimeException: UNKNOWN") + private static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, + OzoneConfiguration conf) + throws IOException { + ContainerSet containerSet = new ContainerSet(1000); + conf.set(HDDS_DATANODE_DIR_KEY, + Paths.get(TEST_DIR, "dfs", "data", "hdds", + RandomStringUtils.randomAlphabetic(4)).toString()); + conf.set(OZONE_METADATA_DIRS, TEST_DIR); + VolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, + StorageVolume.VolumeType.DATA_VOLUME, null); + StateContext context = ContainerTestUtils.getMockContext(dd, conf); + ContainerMetrics metrics = ContainerMetrics.create(conf); + Map handlers = Maps.newHashMap(); + for (ContainerProtos.ContainerType containerType : + ContainerProtos.ContainerType.values()) { + handlers.put(containerType, + Handler.getHandlerForContainerType(containerType, conf, + dd.getUuid().toString(), + containerSet, volumeSet, metrics, + c -> { + })); + } + HddsDispatcher hddsDispatcher = new HddsDispatcher( + conf, containerSet, volumeSet, handlers, context, metrics, null); + hddsDispatcher.setClusterId(scmId.toString()); + return hddsDispatcher; + } + @Test public void testClientServerWithContainerDispatcher() throws Exception { - XceiverServerGrpc server = null; - XceiverClientGrpc client = null; - UUID scmId = UUID.randomUUID(); - try { - Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, - pipeline.getFirstNode() - .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()); - - ContainerSet containerSet = new ContainerSet(1000); - VolumeSet volumeSet = mock(MutableVolumeSet.class); - ContainerMetrics metrics = ContainerMetrics.create(conf); - Map handlers = Maps.newHashMap(); - DatanodeDetails datanodeDetails = randomDatanodeDetails(); - StateContext context = ContainerTestUtils.getMockContext( - datanodeDetails, conf); - - - for (ContainerProtos.ContainerType containerType : - ContainerProtos.ContainerType.values()) { - handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, - context.getParent().getDatanodeDetails().getUuidString(), - containerSet, volumeSet, metrics, - c -> { })); - } - HddsDispatcher dispatcher = new HddsDispatcher( - conf, containerSet, volumeSet, handlers, context, metrics, null); - dispatcher.setClusterId(scmId.toString()); - dispatcher.init(); - - server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher, - caClient); - client = new XceiverClientGrpc(pipeline, conf); - - server.start(); - client.connect(); - - ContainerCommandRequestProto request = - ContainerTestHelper.getCreateContainerRequest( - ContainerTestHelper.getTestContainerID(), pipeline); - ContainerCommandResponseProto response = client.sendCommand(request); - Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); - } finally { - if (client != null) { - client.close(); - } - if (server != null) { - server.stop(); - } - } + DatanodeDetails dd = MockDatanodeDetails.randomDatanodeDetails(); + HddsDispatcher hddsDispatcher = createDispatcher(dd, + UUID.randomUUID(), CONF); + runTestClientServer(1, (pipeline, conf) -> conf + .setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, + pipeline.getFirstNode() + .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()), + XceiverClientGrpc::new, + (dn, conf) -> new XceiverServerGrpc(dd, conf, + hddsDispatcher, caClient), (dn, p) -> { + }); } private static class TestContainerDispatcher implements ContainerDispatcher { From 8df4ddc6b1e6116235e637b532c9af076f5663fc Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 27 Nov 2023 18:38:56 -0800 Subject: [PATCH 09/51] HDDS-9773. Fix to run TestOmSnapshotFileSystem in CI workflow (#5685) --- .../ozone/om/TestOmSnapshotFileSystem.java | 80 +++++++------------ .../ozone/om/TestOmSnapshotFileSystemFso.java | 31 +++++++ .../om/TestOmSnapshotFileSystemLegacy.java | 31 +++++++ 3 files changed, 92 insertions(+), 50 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemFso.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemLegacy.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index 1cd37e736cf..bd384256792 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -90,10 +90,17 @@ import static org.junit.jupiter.api.Assertions.fail; /** - * OmSnapshot file system tests. + * Abstract class for OmSnapshot file system tests. */ @Timeout(120) public abstract class TestOmSnapshotFileSystem { + protected static final String VOLUME_NAME = + "volume" + RandomStringUtils.randomNumeric(5); + protected static final String BUCKET_NAME_FSO = + "bucket-fso-" + RandomStringUtils.randomNumeric(5); + protected static final String BUCKET_NAME_LEGACY = + "bucket-legacy-" + RandomStringUtils.randomNumeric(5); + private static MiniOzoneCluster cluster = null; private static OzoneClient client; private static ObjectStore objectStore; @@ -101,9 +108,6 @@ public abstract class TestOmSnapshotFileSystem { private static OzoneManagerProtocol writeClient; private static OzoneManager ozoneManager; private static String keyPrefix; - private static String volumeName; - private static String bucketNameFso; - private static String bucketNameLegacy; private final String bucketName; private FileSystem fs; private OzoneFileSystem o3fs; @@ -115,26 +119,6 @@ public TestOmSnapshotFileSystem(String bucketName) { this.bucketName = bucketName; } - /** - * OmSnapshot file system tests for FSO. - */ - public static class TestOmSnapshotFileSystemFso - extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemFso() { - super(bucketNameFso); - } - } - - /** - * OmSnapshot file system tests for Legacy. - */ - public static class TestOmSnapshotFileSystemLegacy - extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemLegacy() { - super(bucketNameLegacy); - } - } - @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); @@ -151,17 +135,13 @@ public static void init() throws Exception { writeClient = objectStore.getClientProxy().getOzoneManagerClient(); ozoneManager = cluster.getOzoneManager(); - volumeName = "volume" + RandomStringUtils.randomNumeric(5); - bucketNameFso = "bucket-fso-" + RandomStringUtils.randomNumeric(5); - bucketNameLegacy = "bucket-legacy-" + RandomStringUtils.randomNumeric(5); - - TestDataUtil.createVolume(client, volumeName); - TestDataUtil.createBucket(client, volumeName, + TestDataUtil.createVolume(client, VOLUME_NAME); + TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(FILE_SYSTEM_OPTIMIZED).build(), - bucketNameFso); - TestDataUtil.createBucket(client, volumeName, + BUCKET_NAME_FSO); + TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(LEGACY).build(), - bucketNameLegacy); + BUCKET_NAME_LEGACY); // stop the deletion services so that keys can still be read KeyManagerImpl keyManager = (KeyManagerImpl) ozoneManager.getKeyManager(); @@ -171,7 +151,7 @@ public static void init() throws Exception { @BeforeEach public void setupFsClient() throws IOException { String rootPath = String.format("%s://%s.%s/", - OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); + OzoneConsts.OZONE_URI_SCHEME, bucketName, VOLUME_NAME); // Set the fs.defaultFS and start the filesystem conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); // Set the number of keys to be processed during batch operate. @@ -223,8 +203,8 @@ public void deleteRootDir() @Test // based on TestObjectStoreWithFSO:testListKeysAtDifferentLevels public void testListKeysAtDifferentLevels() throws Exception { - OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - assertEquals(ozoneVolume.getName(), volumeName); + OzoneVolume ozoneVolume = objectStore.getVolume(VOLUME_NAME); + assertEquals(ozoneVolume.getName(), VOLUME_NAME); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); assertEquals(ozoneBucket.getName(), bucketName); @@ -291,7 +271,7 @@ public void testListKeysAtDifferentLevels() throws Exception { deleteSnapshot(snapshotName); String expectedMessage = String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName); + VOLUME_NAME, bucketName, snapshotName); OMException exception = assertThrows(OMException.class, () -> ozoneBucket.listKeys(keyPrefix + "a/", null)); assertEquals(expectedMessage, exception.getMessage()); @@ -383,7 +363,7 @@ private void createKey(OzoneBucket ozoneBucket, String key, int length, // Read using filesystem. String rootPath = String.format("%s://%s.%s/", OZONE_URI_SCHEME, - bucketName, volumeName); + bucketName, VOLUME_NAME); OzoneFileSystem o3fsNew = (OzoneFileSystem) FileSystem .get(new URI(rootPath), conf); FSDataInputStream fsDataInputStream = o3fsNew.open(new Path(key)); @@ -511,21 +491,21 @@ public void testListStatus() throws Exception { () -> fs.listStatus(snapshotRoot1)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName1), exception1.getMessage()); + VOLUME_NAME, bucketName, snapshotName1), exception1.getMessage()); deleteSnapshot(snapshotName2); FileNotFoundException exception2 = assertThrows(FileNotFoundException.class, () -> fs.listStatus(snapshotRoot2)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName2), exception2.getMessage()); + VOLUME_NAME, bucketName, snapshotName2), exception2.getMessage()); deleteSnapshot(snapshotName3); FileNotFoundException exception3 = assertThrows(FileNotFoundException.class, () -> fs.listStatus(snapshotParent3)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName3), exception3.getMessage()); + VOLUME_NAME, bucketName, snapshotName3), exception3.getMessage()); } @Test @@ -560,7 +540,7 @@ public void testListStatusWithIntermediateDir() throws Exception { () -> fs.listStatus(snapshotParent)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); } @Test @@ -596,7 +576,7 @@ public void testGetFileStatus() throws Exception { () -> fs.listStatus(snapshotParent)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); } @Test @@ -639,11 +619,11 @@ public void testReadFileFromSnapshot() throws Exception { () -> fs.open(fileInSnapshot)); assertEquals(String.format("FILE_NOT_FOUND: Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); } private void createAndCommitKey(String keyName) throws IOException { - OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName) + OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(VOLUME_NAME) .setBucketName(bucketName).setKeyName(keyName) .setAcls(Collections.emptyList()) .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) @@ -688,7 +668,7 @@ public void testListStatusOnRoot() throws Exception { () -> fs.listStatus(snapshotRoot)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); } /** @@ -745,19 +725,19 @@ public void testListStatusOnLargeDirectory() throws Exception { () -> fs.listStatus(snapshotRoot)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - volumeName, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); } private String createSnapshot(String snapshotName) throws IOException, InterruptedException, TimeoutException { // create snapshot - writeClient.createSnapshot(volumeName, bucketName, snapshotName); + writeClient.createSnapshot(VOLUME_NAME, bucketName, snapshotName); // wait till the snapshot directory exists SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() - .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); + .get(SnapshotInfo.getTableKey(VOLUME_NAME, bucketName, snapshotName)); String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), @@ -767,6 +747,6 @@ private String createSnapshot(String snapshotName) } private void deleteSnapshot(String snapshotName) throws IOException { - writeClient.deleteSnapshot(volumeName, bucketName, snapshotName); + writeClient.deleteSnapshot(VOLUME_NAME, bucketName, snapshotName); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemFso.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemFso.java new file mode 100644 index 00000000000..66d39516020 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemFso.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import org.junit.jupiter.api.Timeout; + +/** + * OmSnapshot file system tests for FSO. + */ +@Timeout(120) +public class TestOmSnapshotFileSystemFso extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemFso() { + super(BUCKET_NAME_FSO); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemLegacy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemLegacy.java new file mode 100644 index 00000000000..86682b2cbc1 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystemLegacy.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import org.junit.jupiter.api.Timeout; + +/** + * OmSnapshot file system tests for Legacy. + */ +@Timeout(120) +public class TestOmSnapshotFileSystemLegacy extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemLegacy() { + super(BUCKET_NAME_LEGACY); + } +} From ae58db36bc65a568f27873aa8978d58962540e01 Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Tue, 28 Nov 2023 14:29:00 +0530 Subject: [PATCH 10/51] HDDS-9769. SCM's FinalizationStateManager#finalizeLayoutFeature Ratis call should be idempotent. (#5670) --- .../upgrade/AbstractLayoutVersionManager.java | 20 +++++++++---------- .../TestAbstractLayoutVersionManager.java | 16 ++++++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java index e389f506058..3e09ae2b076 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java @@ -129,20 +129,20 @@ public void finalized(T layoutFeature) { LOG.info("Finalization is complete."); } } else { - String msgStart = ""; + String versionMsg = "Software layout version: " + softwareLayoutVersion + + ", Metadata layout version: " + metadataLayoutVersion + + ", Feature Layout version: " + layoutFeature.layoutVersion() + + "."; + if (layoutFeature.layoutVersion() <= metadataLayoutVersion) { - msgStart = "Finalize attempt on a layoutFeature which has already " - + "been finalized."; + LOG.info("Finalize attempt on a layoutFeature which has already " + + "been finalized. " + versionMsg + " This can happen when " + + "Raft Log is replayed during service restart."); } else { - msgStart = + throw new IllegalArgumentException( "Finalize attempt on a layoutFeature that is newer than the " + - "next feature to be finalized."; + "next feature to be finalized. " + versionMsg); } - - throw new IllegalArgumentException( - msgStart + " Software layout version: " + softwareLayoutVersion - + " Metadata layout version: " + metadataLayoutVersion - + " Feature Layout version: " + layoutFeature.layoutVersion()); } } finally { lock.writeLock().unlock(); diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java index fcc76bed642..909302c8a31 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -122,13 +122,19 @@ public void testFeatureFinalizationFailsIfTheFinalizedFeatureIsNotTheNext() } @Test - public void testFeatureFinalizationFailsIfFeatureIsAlreadyFinalized() + public void testFeatureFinalizationIfFeatureIsAlreadyFinalized() throws IOException { + /* + * Feature finalization call is idempotent, it should not have any + * side effects even if it's executed again. + */ LayoutFeature[] lfs = getTestLayoutFeatures(3); - versionManager.init(1, lfs); - - assertThrows(IllegalArgumentException.class, - () -> versionManager.finalized(lfs[0])); + versionManager.init(2, lfs); + assertEquals(2, versionManager.getMetadataLayoutVersion()); + versionManager.finalized(lfs[0]); + assertEquals(2, versionManager.getMetadataLayoutVersion()); + versionManager.finalized(lfs[1]); + assertEquals(2, versionManager.getMetadataLayoutVersion()); } @Test From b59366dcead37b3c0ed96255652a7b7471dc0971 Mon Sep 17 00:00:00 2001 From: XiChen <32928346+xichen01@users.noreply.github.com> Date: Tue, 28 Nov 2023 18:39:24 +0800 Subject: [PATCH 11/51] HDDS-9718. Add performance audit logging for S3G (#5644) --- .../hadoop/ozone/audit/AuditLogger.java | 72 +++++++++++ .../hadoop/ozone/audit/AuditMessage.java | 22 +++- .../apache/hadoop/ozone/audit/S3GAction.java | 1 + .../ozone/s3/endpoint/BucketEndpoint.java | 12 +- .../ozone/s3/endpoint/EndpointBase.java | 8 ++ .../ozone/s3/endpoint/ObjectEndpoint.java | 122 +++++++++++------- .../s3/endpoint/ObjectEndpointStreaming.java | 32 ++++- .../ozone/s3/metrics/S3GatewayMetrics.java | 43 +++--- 8 files changed, 233 insertions(+), 79 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java index b27fc78771a..f4f8ba78537 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java @@ -30,6 +30,7 @@ import java.util.HashSet; import java.util.Locale; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -132,4 +133,75 @@ private boolean shouldLogAtDebug(AuditMessage auditMessage) { return debugCmdSetRef.get() .contains(auditMessage.getOp().toLowerCase(Locale.ROOT)); } + + /** + * Utility class for building performance log strings. + */ + public static class PerformanceStringBuilder { + private final StringBuilder builder = new StringBuilder(128).append('{'); + /** + * Appends metadata operation latency in milliseconds. + * @param nanos Latency in nanoseconds. + */ + public void appendMetaLatencyNanos(long nanos) { + append("metaLatencyMs", TimeUnit.NANOSECONDS.toMillis(nanos)); + } + + /** + * Appends whole operation latency in milliseconds. + * @param nanos Latency in nanoseconds. + */ + public void appendOpLatencyNanos(long nanos) { + append("opLatencyMs", TimeUnit.NANOSECONDS.toMillis(nanos)); + } + + /** + * Appends the size in bytes. + * @param bytes Size in bytes. + */ + public void appendSizeBytes(long bytes) { + append("sizeByte", bytes); + } + + /** + * Appends the count. + * @param count The count value to be appended. + */ + public void appendCount(long count) { + append("count", count); + } + + /** + * Appends a stream mode flag. + */ + public void appendStreamMode() { + append("streamMode", "true"); + } + + private void append(String name, long value) { + append(name, String.valueOf(value)); + } + + /** + * Appends a name-value pair to the log string. + * @param name Name of the metric. + * @param value Value of the metric. + */ + private void append(String name, String value) { + builder.append(name) + .append('=') + .append(value) + .append(", "); + } + + public String build() { + final int length = builder.length(); + if (length < 2) { + return "{}"; + } + builder.setCharAt(length - 2, '}'); + builder.setLength(length - 1); + return builder.toString(); + } + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java index 85fa7986b96..bff05f024de 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java @@ -34,13 +34,14 @@ public final class AuditMessage implements Message { private final Throwable throwable; private AuditMessage(String user, String ip, String op, - Map params, String ret, Throwable throwable) { + Map params, String ret, Throwable throwable, + String performance) { this.user = user; this.ip = ip; this.op = op; this.params = params; this.ret = ret; - this.message = formMessage(user, ip, op, params, ret); + this.message = formMessage(user, ip, op, params, ret, performance); this.throwable = throwable; } @@ -78,6 +79,7 @@ public static class Builder { private String op; private Map params; private String ret; + private String performance; public Builder setUser(String usr) { this.user = usr; @@ -109,15 +111,23 @@ public Builder withException(Throwable ex) { return this; } + public Builder setPerformance(String perf) { + this.performance = perf; + return this; + } + public AuditMessage build() { - return new AuditMessage(user, ip, op, params, ret, throwable); + return new AuditMessage(user, ip, op, params, ret, throwable, + performance); } } private String formMessage(String userStr, String ipStr, String opStr, - Map paramsMap, String retStr) { + Map paramsMap, String retStr, + String performanceMap) { + String perf = performanceMap != null && !performanceMap.isEmpty() + ? " | perf=" + performanceMap : ""; return "user=" + userStr + " | ip=" + ipStr + " | " + "op=" + opStr - + " " + paramsMap + " | " + "ret=" + retStr; - + + " " + paramsMap + " | ret=" + retStr + perf; } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java index dcf116ea0f8..20c2f4c6275 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java @@ -36,6 +36,7 @@ public enum S3GAction implements AuditAction { //ObjectEndpoint CREATE_MULTIPART_KEY, + CREATE_MULTIPART_KEY_BY_COPY, COPY_OBJECT, CREATE_KEY, LIST_PARTS, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 6ab3a4ba7fb..910b0026e31 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -69,6 +69,7 @@ import java.util.List; import java.util.Set; +import static org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED; @@ -113,6 +114,8 @@ public Response get( @Context HttpHeaders hh) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.GET_BUCKET; + PerformanceStringBuilder perf = new PerformanceStringBuilder(); + Iterator ozoneKeyIterator; ContinueToken decodedToken = ContinueToken.decodeFromString(continueToken); @@ -264,12 +267,15 @@ public Response get( response.setTruncated(false); } - AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters())); int keyCount = response.getCommonPrefixes().size() + response.getContents().size(); - getMetrics().updateGetBucketSuccessStats(startNanos); + long opLatencyNs = + getMetrics().updateGetBucketSuccessStats(startNanos); getMetrics().incListKeyCount(keyCount); + perf.appendCount(keyCount); + perf.appendOpLatencyNanos(opLatencyNs); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, + getAuditParameters(), perf.build())); response.setKeyCount(keyCount); return Response.ok(response).build(); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 05b7a62c062..abda4678dca 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -353,6 +353,14 @@ public AuditMessage buildAuditMessageForSuccess(AuditAction op, return builder.build(); } + public AuditMessage buildAuditMessageForSuccess(AuditAction op, + Map auditMap, String performance) { + AuditMessage.Builder builder = auditMessageBaseBuilder(op, auditMap) + .withResult(AuditEventStatus.SUCCESS); + builder.setPerformance(performance); + return builder.build(); + } + @Override public AuditMessage buildAuditMessageForFailure(AuditAction op, Map auditMap, Throwable throwable) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index ad4bbebeebc..b607b1c5cff 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -112,6 +112,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT; +import static org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY; @@ -218,16 +219,21 @@ public Response put( InputStream body) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.CREATE_KEY; - boolean auditSuccess = true; + PerformanceStringBuilder perf = new PerformanceStringBuilder(); + String copyHeader = null, storageType = null; try { OzoneVolume volume = getVolume(); if (uploadID != null && !uploadID.equals("")) { - s3GAction = S3GAction.CREATE_MULTIPART_KEY; + if (headers.getHeaderString(COPY_SOURCE_HEADER) == null) { + s3GAction = S3GAction.CREATE_MULTIPART_KEY; + } else { + s3GAction = S3GAction.CREATE_MULTIPART_KEY_BY_COPY; + } // If uploadID is specified, it is a request for upload part return createMultipartKey(volume, bucketName, keyPath, length, - partNumber, uploadID, body); + partNumber, uploadID, body, perf); } copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER); @@ -251,7 +257,7 @@ public Response put( s3GAction = S3GAction.COPY_OBJECT; CopyObjectResponse copyObjectResponse = copyObject(volume, copyHeader, bucketName, keyPath, replicationConfig, - storageTypeDefault); + storageTypeDefault, perf); return Response.status(Status.OK).entity(copyObjectResponse).header( "Connection", "close").build(); } @@ -270,6 +276,9 @@ public Response put( s3GAction = S3GAction.CREATE_DIRECTORY; getClientProtocol() .createDirectory(volume.getName(), bucketName, keyPath); + long metadataLatencyNs = + getMetrics().updatePutKeyMetadataStats(startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); return Response.ok().status(HttpStatus.SC_OK).build(); } @@ -297,17 +306,19 @@ public Response put( long putLength; String eTag = null; if (datastreamEnabled && !enableEC && length > datastreamMinLength) { - getMetrics().updatePutKeyMetadataStats(startNanos); + perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, chunkSize, - customMetadata, (DigestInputStream) body); + customMetadata, (DigestInputStream) body, perf); eTag = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { try (OzoneOutputStream output = getClientProtocol().createKey( volume.getName(), bucketName, keyPath, length, replicationConfig, customMetadata)) { - getMetrics().updatePutKeyMetadataStats(startNanos); + long metadataLatencyNs = + getMetrics().updatePutKeyMetadataStats(startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); putLength = IOUtils.copyLarge(body, output); eTag = DatatypeConverter.printHexBinary( ((DigestInputStream) body).getMessageDigest().digest()) @@ -315,8 +326,8 @@ public Response put( output.getMetadata().put(ETAG, eTag); } } - getMetrics().incPutKeySuccessLength(putLength); + perf.appendSizeBytes(putLength); return Response.ok() .header(ETAG, wrapInQuotes(eTag)) .status(HttpStatus.SC_OK) @@ -356,9 +367,10 @@ public Response put( throw ex; } finally { if (auditSuccess) { - AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); - getMetrics().updateCreateKeySuccessStats(startNanos); + long opLatencyNs = getMetrics().updateCreateKeySuccessStats(startNanos); + perf.appendOpLatencyNanos(opLatencyNs); + AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, + getAuditParameters(), perf.build())); } } } @@ -372,6 +384,7 @@ public Response put( * https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadListParts.html * for more details. */ + @SuppressWarnings("checkstyle:MethodLength") @GET public Response get( @PathParam("bucket") String bucketName, @@ -383,15 +396,17 @@ public Response get( throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.GET_KEY; - boolean auditSuccess = true; - + PerformanceStringBuilder perf = new PerformanceStringBuilder(); try { if (uploadId != null) { // When we have uploadId, this is the request for list Parts. s3GAction = S3GAction.LIST_PARTS; int partMarker = parsePartNumberMarker(partNumberMarker); - return listParts(bucketName, keyPath, uploadId, - partMarker, maxParts); + Response response = listParts(bucketName, keyPath, uploadId, + partMarker, maxParts, perf); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, + getAuditParameters(), perf.build())); + return response; } OzoneKeyDetails keyDetails = (partNumber != 0) ? @@ -424,8 +439,12 @@ public Response get( try (OzoneInputStream key = keyDetails.getContent()) { long readLength = IOUtils.copyLarge(key, dest); getMetrics().incGetKeySuccessLength(readLength); + perf.appendSizeBytes(readLength); } - getMetrics().updateGetKeySuccessStats(startNanos); + long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); + perf.appendOpLatencyNanos(opLatencyNs); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, + getAuditParameters(), perf.build())); }; responseBuilder = Response .ok(output) @@ -444,8 +463,12 @@ public Response get( long readLength = IOUtils.copyLarge(ozoneInputStream, dest, 0, copyLength, new byte[bufferSize]); getMetrics().incGetKeySuccessLength(readLength); + perf.appendSizeBytes(readLength); } - getMetrics().updateGetKeySuccessStats(startNanos); + long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); + perf.appendOpLatencyNanos(opLatencyNs); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, + getAuditParameters(), perf.build())); }; responseBuilder = Response .status(Status.PARTIAL_CONTENT) @@ -486,10 +509,11 @@ public Response get( } } addLastModifiedDate(responseBuilder, keyDetails); - getMetrics().updateGetKeyMetadataStats(startNanos); + long metadataLatencyNs = + getMetrics().updateGetKeyMetadataStats(startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); return responseBuilder.build(); } catch (OMException ex) { - auditSuccess = false; AUDIT.logReadFailure( buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex) ); @@ -508,17 +532,10 @@ public Response get( throw ex; } } catch (Exception ex) { - auditSuccess = false; AUDIT.logReadFailure( buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex) ); throw ex; - } finally { - if (auditSuccess) { - AUDIT.logReadSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters()) - ); - } } } @@ -847,10 +864,10 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket, } } - @SuppressWarnings("checkstyle:MethodLength") + @SuppressWarnings({"checkstyle:MethodLength", "checkstyle:ParameterNumber"}) private Response createMultipartKey(OzoneVolume volume, String bucket, - String key, long length, int partNumber, - String uploadID, InputStream body) + String key, long length, int partNumber, String uploadID, + InputStream body, PerformanceStringBuilder perf) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); String copyHeader = null; @@ -880,16 +897,17 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, } if (datastreamEnabled && !enableEC && copyHeader == null) { - getMetrics().updatePutKeyMetadataStats(startNanos); + perf.appendStreamMode(); return ObjectEndpointStreaming .createMultipartKey(ozoneBucket, key, length, partNumber, - uploadID, chunkSize, (DigestInputStream) body); + uploadID, chunkSize, (DigestInputStream) body, perf); } // OmMultipartCommitUploadPartInfo can only be gotten after the // OzoneOutputStream is closed, so we need to save the KeyOutputStream // in the OzoneOutputStream and use it to get the // OmMultipartCommitUploadPartInfo after OzoneOutputStream is closed. KeyOutputStream keyOutputStream = null; + long metadataLatencyNs; if (copyHeader != null) { Pair result = parseSourceHeader(copyHeader); String sourceBucket = result.getLeft(); @@ -933,7 +951,8 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, try (OzoneOutputStream ozoneOutputStream = getClientProtocol() .createMultipartKey(volume.getName(), bucket, key, length, partNumber, uploadID)) { - getMetrics().updateCopyKeyMetadataStats(startNanos); + metadataLatencyNs = + getMetrics().updateCopyKeyMetadataStats(startNanos); copyLength = IOUtils.copyLarge( sourceObject, ozoneOutputStream, 0, length); keyOutputStream = ozoneOutputStream.getKeyOutputStream(); @@ -942,19 +961,22 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, try (OzoneOutputStream ozoneOutputStream = getClientProtocol() .createMultipartKey(volume.getName(), bucket, key, length, partNumber, uploadID)) { - getMetrics().updateCopyKeyMetadataStats(startNanos); + metadataLatencyNs = + getMetrics().updateCopyKeyMetadataStats(startNanos); copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream); keyOutputStream = ozoneOutputStream.getKeyOutputStream(); } } getMetrics().incCopyObjectSuccessLength(copyLength); + perf.appendSizeBytes(copyLength); } } else { long putLength; try (OzoneOutputStream ozoneOutputStream = getClientProtocol() .createMultipartKey(volume.getName(), bucket, key, length, partNumber, uploadID)) { - getMetrics().updatePutKeyMetadataStats(startNanos); + metadataLatencyNs = + getMetrics().updatePutKeyMetadataStats(startNanos); putLength = IOUtils.copyLarge(body, ozoneOutputStream); ((KeyMetadataAware)ozoneOutputStream.getOutputStream()) .getMetadata().put(ETAG, DatatypeConverter.printHexBinary( @@ -964,7 +986,9 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, = ozoneOutputStream.getKeyOutputStream(); } getMetrics().incPutKeySuccessLength(putLength); + perf.appendSizeBytes(putLength); } + perf.appendMetaLatencyNanos(metadataLatencyNs); assert keyOutputStream != null; OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo = @@ -1012,7 +1036,8 @@ private Response createMultipartKey(OzoneVolume volume, String bucket, * @throws OS3Exception */ private Response listParts(String bucket, String key, String uploadID, - int partNumberMarker, int maxParts) throws IOException, OS3Exception { + int partNumberMarker, int maxParts, PerformanceStringBuilder perf) + throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); ListPartsResponse listPartsResponse = new ListPartsResponse(); try { @@ -1055,7 +1080,9 @@ private Response listParts(String bucket, String key, String uploadID, } throw ex; } - getMetrics().updateListPartsSuccessStats(startNanos); + long opLatencyNs = getMetrics().updateListPartsSuccessStats(startNanos); + perf.appendCount(listPartsResponse.getPartList().size()); + perf.appendOpLatencyNanos(opLatencyNs); return Response.status(Status.OK).entity(listPartsResponse).build(); } @@ -1069,33 +1096,40 @@ public void setContext(ContainerRequestContext context) { this.context = context; } + @SuppressWarnings("checkstyle:ParameterNumber") void copy(OzoneVolume volume, InputStream src, long srcKeyLen, String destKey, String destBucket, ReplicationConfig replication, - Map metadata) throws IOException { + Map metadata, + PerformanceStringBuilder perf, long startNanos) + throws IOException { long copyLength; if (datastreamEnabled && !(replication != null && replication.getReplicationType() == EC) && srcKeyLen > datastreamMinLength) { + perf.appendStreamMode(); copyLength = ObjectEndpointStreaming .copyKeyWithStream(volume.getBucket(destBucket), destKey, srcKeyLen, - chunkSize, replication, metadata, src); + chunkSize, replication, metadata, src, perf, startNanos); } else { try (OzoneOutputStream dest = getClientProtocol() .createKey(volume.getName(), destBucket, destKey, srcKeyLen, replication, metadata)) { + long metadataLatencyNs = + getMetrics().updateCopyKeyMetadataStats(startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); copyLength = IOUtils.copyLarge(src, dest); } } getMetrics().incCopyObjectSuccessLength(copyLength); + perf.appendSizeBytes(copyLength); } + @SuppressWarnings("checkstyle:ParameterNumber") private CopyObjectResponse copyObject(OzoneVolume volume, - String copyHeader, - String destBucket, - String destkey, - ReplicationConfig replicationConfig, - boolean storageTypeDefault) + String copyHeader, String destBucket, String destkey, + ReplicationConfig replicationConfig, boolean storageTypeDefault, + PerformanceStringBuilder perf) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); Pair result = parseSourceHeader(copyHeader); @@ -1138,7 +1172,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume, sourceBucket, sourceKey)) { getMetrics().updateCopyKeyMetadataStats(startNanos); copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig, - sourceKeyDetails.getMetadata()); + sourceKeyDetails.getMetadata(), perf, startNanos); } final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index b536b3248b8..dbc7f374a9a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ import java.security.DigestInputStream; import java.util.Map; +import static org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_UPLOAD; @@ -54,16 +56,17 @@ final class ObjectEndpointStreaming { private ObjectEndpointStreaming() { } + @SuppressWarnings("checkstyle:ParameterNumber") public static Pair put( OzoneBucket bucket, String keyPath, long length, ReplicationConfig replicationConfig, int chunkSize, Map keyMetadata, - DigestInputStream body) + DigestInputStream body, PerformanceStringBuilder perf) throws IOException, OS3Exception { try { return putKeyWithStream(bucket, keyPath, - length, chunkSize, replicationConfig, keyMetadata, body); + length, chunkSize, replicationConfig, keyMetadata, body, perf); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); if (ex instanceof OMException) { @@ -86,6 +89,7 @@ public static Pair put( } } + @SuppressWarnings("checkstyle:ParameterNumber") public static Pair putKeyWithStream( OzoneBucket bucket, String keyPath, @@ -93,20 +97,25 @@ public static Pair putKeyWithStream( int bufferSize, ReplicationConfig replicationConfig, Map keyMetadata, - DigestInputStream body) + DigestInputStream body, PerformanceStringBuilder perf) throws IOException { + S3GatewayMetrics metrics = S3GatewayMetrics.create(); + long startNanos = Time.monotonicNowNanos(); long writeLen; String eTag; try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath, length, replicationConfig, keyMetadata)) { + long metadataLatencyNs = metrics.updatePutKeyMetadataStats(startNanos); writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); eTag = DatatypeConverter.printHexBinary(body.getMessageDigest().digest()) .toLowerCase(); + perf.appendMetaLatencyNanos(metadataLatencyNs); ((KeyMetadataAware)streamOutput).getMetadata().put("ETag", eTag); } return Pair.of(eTag, writeLen); } + @SuppressWarnings("checkstyle:ParameterNumber") public static long copyKeyWithStream( OzoneBucket bucket, String keyPath, @@ -114,10 +123,15 @@ public static long copyKeyWithStream( int bufferSize, ReplicationConfig replicationConfig, Map keyMetadata, - InputStream body) throws IOException { + InputStream body, PerformanceStringBuilder perf, long startNanos) + throws IOException { long writeLen = 0; + S3GatewayMetrics metrics = S3GatewayMetrics.create(); try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath, length, replicationConfig, keyMetadata)) { + long metadataLatencyNs = + metrics.updateCopyKeyMetadataStats(startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); } return writeLen; @@ -141,11 +155,12 @@ private static long writeToStreamOutput(OzoneDataStreamOutput streamOutput, return n; } + @SuppressWarnings("checkstyle:ParameterNumber") public static Response createMultipartKey(OzoneBucket ozoneBucket, String key, - long length, int partNumber, - String uploadID, int chunkSize, - DigestInputStream body) + long length, int partNumber, String uploadID, int chunkSize, + DigestInputStream body, PerformanceStringBuilder perf) throws IOException, OS3Exception { + long startNanos = Time.monotonicNowNanos(); String eTag; S3GatewayMetrics metrics = S3GatewayMetrics.create(); // OmMultipartCommitUploadPartInfo can only be gotten after the @@ -156,12 +171,15 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key, try { try (OzoneDataStreamOutput streamOutput = ozoneBucket .createMultipartStreamKey(key, length, partNumber, uploadID)) { + long metadataLatencyNs = metrics.updatePutKeyMetadataStats(startNanos); long putLength = writeToStreamOutput(streamOutput, body, chunkSize, length); eTag = DatatypeConverter.printHexBinary( body.getMessageDigest().digest()).toLowerCase(); ((KeyMetadataAware)streamOutput).getMetadata().put("ETag", eTag); metrics.incPutKeySuccessLength(putLength); + perf.appendMetaLatencyNanos(metadataLatencyNs); + perf.appendSizeBytes(putLength); keyDataStreamOutput = streamOutput.getKeyDataStreamOutput(); } } catch (OMException ex) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java index b18b9f3354d..10b7b167b9d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java @@ -362,9 +362,9 @@ public void getMetrics(MetricsCollector collector, boolean all) { // INC and UPDATE // BucketEndpoint - public void updateGetBucketSuccessStats(long startNanos) { + public long updateGetBucketSuccessStats(long startNanos) { getBucketSuccess.incr(); - getBucketSuccessLatencyNs.add(Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(getBucketSuccessLatencyNs, startNanos); } public void updateGetBucketFailureStats(long startNanos) { @@ -447,10 +447,9 @@ public void updateListS3BucketsFailureStats(long startNanos) { // ObjectEndpoint - public void updateCreateMultipartKeySuccessStats(long startNanos) { + public long updateCreateMultipartKeySuccessStats(long startNanos) { createMultipartKeySuccess.incr(); - createMultipartKeySuccessLatencyNs.add( - Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(createMultipartKeySuccessLatencyNs, startNanos); } public void updateCreateMultipartKeyFailureStats(long startNanos) { @@ -459,9 +458,9 @@ public void updateCreateMultipartKeyFailureStats(long startNanos) { Time.monotonicNowNanos() - startNanos); } - public void updateCopyObjectSuccessStats(long startNanos) { + public long updateCopyObjectSuccessStats(long startNanos) { copyObjectSuccess.incr(); - copyObjectSuccessLatencyNs.add(Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(copyObjectSuccessLatencyNs, startNanos); } public void updateCopyObjectFailureStats(long startNanos) { @@ -469,9 +468,9 @@ public void updateCopyObjectFailureStats(long startNanos) { copyObjectFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos); } - public void updateCreateKeySuccessStats(long startNanos) { + public long updateCreateKeySuccessStats(long startNanos) { createKeySuccess.incr(); - createKeySuccessLatencyNs.add(Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(createKeySuccessLatencyNs, startNanos); } public void updateCreateKeyFailureStats(long startNanos) { @@ -479,9 +478,9 @@ public void updateCreateKeyFailureStats(long startNanos) { createKeyFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos); } - public void updateListPartsSuccessStats(long startNanos) { + public long updateListPartsSuccessStats(long startNanos) { listPartsSuccess.incr(); - listPartsSuccessLatencyNs.add(Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(listPartsSuccessLatencyNs, startNanos); } public void updateListPartsFailureStats(long startNanos) { @@ -489,9 +488,9 @@ public void updateListPartsFailureStats(long startNanos) { listPartsFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos); } - public void updateGetKeySuccessStats(long startNanos) { + public long updateGetKeySuccessStats(long startNanos) { getKeySuccess.incr(); - getKeySuccessLatencyNs.add(Time.monotonicNowNanos() - startNanos); + return updateAndGetStats(getKeySuccessLatencyNs, startNanos); } public void updateGetKeyFailureStats(long startNanos) { @@ -555,16 +554,16 @@ public void updateDeleteKeyFailureStats(long startNanos) { deleteKeyFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos); } - public void updateGetKeyMetadataStats(long startNanos) { - getKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos); + public long updateGetKeyMetadataStats(long startNanos) { + return updateAndGetStats(getKeyMetadataLatencyNs, startNanos); } - public void updateCopyKeyMetadataStats(long startNanos) { - copyKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos); + public long updateCopyKeyMetadataStats(long startNanos) { + return updateAndGetStats(copyKeyMetadataLatencyNs, startNanos); } - public void updatePutKeyMetadataStats(long startNanos) { - putKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos); + public long updatePutKeyMetadataStats(long startNanos) { + return updateAndGetStats(putKeyMetadataLatencyNs, startNanos); } public void incCopyObjectSuccessLength(long bytes) { @@ -719,4 +718,10 @@ public long getHeadKeyFailure() { public long getListS3BucketsFailure() { return listS3BucketsFailure.value(); } + + private long updateAndGetStats(MutableRate metric, long startNanos) { + long value = Time.monotonicNowNanos() - startNanos; + metric.add(value); + return value; + } } From 3aca2952186d5539dc19329308c8ad7040133856 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Tue, 28 Nov 2023 16:11:40 +0530 Subject: [PATCH 12/51] HDDS-9702. Improve logging when Recon gets a full update from OM (#5612) --- .../ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java index 8d744dd7769..4742e8c0c45 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java @@ -523,7 +523,8 @@ public boolean syncDataFromOM() { Thread.currentThread().interrupt(); } catch (Exception e) { metrics.incrNumDeltaRequestsFailed(); - LOG.warn("Unable to get and apply delta updates from OM.", e); + LOG.warn("Unable to get and apply delta updates from OM.", + e.getMessage()); fullSnapshot = true; } } From d83899392227cb41ebabacdee2dc17b78fd5efe0 Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Tue, 28 Nov 2023 16:15:13 +0530 Subject: [PATCH 13/51] HDDS-9783. Unregister MBean after test execution in TestAbstractLayoutVersionManager. (#5687) --- .../ozone/upgrade/TestAbstractLayoutVersionManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java index 909302c8a31..9114da5d41b 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -29,6 +29,7 @@ import java.lang.management.ManagementFactory; import java.util.Iterator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.MockitoAnnotations; @@ -50,6 +51,11 @@ public void setup() { MockitoAnnotations.initMocks(this); } + @AfterEach + public void close() { + versionManager.close(); + } + @Test public void testInitializationWithFeaturesToBeFinalized() throws Exception { versionManager.init(1, getTestLayoutFeatures(3)); From d1961391afa5ce98ecf49985558a39d873b996b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:07:08 +0100 Subject: [PATCH 14/51] HDDS-9787. Bump jersey2.version to 2.41 (#5683) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d74118d02c4..4ebfae8215f 100644 --- a/pom.xml +++ b/pom.xml @@ -160,7 +160,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.19.4 - 2.34 + 2.41 1.9.13 From ee6a4ad99e212c9809566c9f6a246b5e60007ae4 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:20:04 +0100 Subject: [PATCH 15/51] HDDS-9674. Read from non-datanode host does not consider topology (#5610) --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 62 +++++++------------ .../scm/server/SCMBlockProtocolServer.java | 40 +++++++++--- .../scm/server/StorageContainerManager.java | 34 +++++++++- .../server/TestSCMBlockProtocolServer.java | 45 ++++++++++---- 4 files changed, 123 insertions(+), 58 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 3103d5a7d4a..167b25afd01 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -54,9 +54,6 @@ import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.metrics2.util.MBeans; -import org.apache.hadoop.net.CachedDNSToSwitchMapping; -import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.net.TableMapping; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; @@ -65,7 +62,6 @@ import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.slf4j.Logger; @@ -88,6 +84,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTP; @@ -123,7 +120,7 @@ public class SCMNodeManager implements NodeManager { private ObjectName nmInfoBean; private final SCMStorageConfig scmStorageConfig; private final NetworkTopology clusterMap; - private final DNSToSwitchMapping dnsToSwitchMapping; + private final Function nodeResolver; private final boolean useHostname; private final Map> dnsToUuidMap = new ConcurrentHashMap<>(); private final int numPipelinesPerMetadataVolume; @@ -142,12 +139,25 @@ public class SCMNodeManager implements NodeManager { /** * Constructs SCM machine Manager. */ - public SCMNodeManager(OzoneConfiguration conf, - SCMStorageConfig scmStorageConfig, - EventPublisher eventPublisher, - NetworkTopology networkTopology, - SCMContext scmContext, - HDDSLayoutVersionManager layoutVersionManager) { + public SCMNodeManager( + OzoneConfiguration conf, + SCMStorageConfig scmStorageConfig, + EventPublisher eventPublisher, + NetworkTopology networkTopology, + SCMContext scmContext, + HDDSLayoutVersionManager layoutVersionManager) { + this(conf, scmStorageConfig, eventPublisher, networkTopology, scmContext, + layoutVersionManager, hostname -> null); + } + + public SCMNodeManager( + OzoneConfiguration conf, + SCMStorageConfig scmStorageConfig, + EventPublisher eventPublisher, + NetworkTopology networkTopology, + SCMContext scmContext, + HDDSLayoutVersionManager layoutVersionManager, + Function nodeResolver) { this.scmNodeEventPublisher = eventPublisher; this.nodeStateManager = new NodeStateManager(conf, eventPublisher, layoutVersionManager, scmContext); @@ -159,15 +169,7 @@ public SCMNodeManager(OzoneConfiguration conf, registerMXBean(); this.metrics = SCMNodeMetrics.create(this); this.clusterMap = networkTopology; - Class dnsToSwitchMappingClass = - conf.getClass( - DFSConfigKeysLegacy.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, - TableMapping.class, DNSToSwitchMapping.class); - DNSToSwitchMapping newInstance = ReflectionUtils.newInstance( - dnsToSwitchMappingClass, conf); - this.dnsToSwitchMapping = - ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance - : new CachedDNSToSwitchMapping(newInstance)); + this.nodeResolver = nodeResolver; this.useHostname = conf.getBoolean( DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME, DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); @@ -383,7 +385,8 @@ public RegisteredCommand register( final String ipAddress = datanodeDetails.getIpAddress(); final String hostName = datanodeDetails.getHostName(); datanodeDetails.setNetworkName(datanodeDetails.getUuidString()); - String networkLocation = nodeResolve(useHostname ? hostName : ipAddress); + String networkLocation = nodeResolver.apply( + useHostname ? hostName : ipAddress); if (networkLocation != null) { datanodeDetails.setNetworkLocation(networkLocation); } @@ -1412,23 +1415,6 @@ public long getLastHeartbeat(DatanodeDetails datanodeDetails) { } } - private String nodeResolve(String hostname) { - List hosts = new ArrayList<>(1); - hosts.add(hostname); - List resolvedHosts = dnsToSwitchMapping.resolve(hosts); - if (resolvedHosts != null && !resolvedHosts.isEmpty()) { - String location = resolvedHosts.get(0); - if (LOG.isDebugEnabled()) { - LOG.debug("Resolve datanode {} return location {}", hostname, location); - } - return location; - } else { - LOG.error("Node {} Resolution failed. Please make sure that DNS table " + - "mapping or configured mapping is functional.", hostname); - return null; - } - } - /** * Test utility to stop heartbeat check process. * diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 8d29d5eeb80..d2d8d0a63a0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -42,7 +42,9 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.DeleteBlockResult; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.net.InnerNode; import org.apache.hadoop.hdds.scm.net.Node; +import org.apache.hadoop.hdds.scm.net.NodeImpl; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; @@ -69,6 +71,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY; import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.IO_EXCEPTION; +import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT; import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; @@ -345,14 +348,13 @@ public List sortDatanodes(List nodes, auditMap.put("nodes", String.valueOf(nodes)); try { NodeManager nodeManager = scm.getScmNodeManager(); - Node client = null; - List possibleClients = - nodeManager.getNodesByAddress(clientMachine); - if (possibleClients.size() > 0) { - client = possibleClients.get(0); + Node client = getDatanode(clientMachine); + // not datanode + if (client == null) { + client = getOtherNode(clientMachine); } - List nodeList = new ArrayList(); - nodes.stream().forEach(uuid -> { + List nodeList = new ArrayList<>(); + nodes.forEach(uuid -> { DatanodeDetails node = nodeManager.getNodeByUuid(uuid); if (node != null) { nodeList.add(node); @@ -377,6 +379,30 @@ public List sortDatanodes(List nodes, } } + private Node getDatanode(String clientMachine) { + List datanodes = scm.getScmNodeManager() + .getNodesByAddress(clientMachine); + return !datanodes.isEmpty() ? datanodes.get(0) : null; + } + + private Node getOtherNode(String clientMachine) { + try { + String clientLocation = scm.resolveNodeLocation(clientMachine); + if (clientLocation != null) { + Node rack = scm.getClusterMap().getNode(clientLocation); + if (rack instanceof InnerNode) { + return new NodeImpl(clientMachine, clientLocation, + (InnerNode) rack, rack.getLevel() + 1, + NODE_COST_DEFAULT); + } + } + } catch (Exception e) { + LOG.info("Could not resolve client {}: {}", + clientMachine, e.getMessage()); + } + return null; + } + @Override public AuditMessage buildAuditMessageForSuccess( AuditAction op, Map auditMap) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 70636215b2d..edba7bb6d34 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -25,10 +25,12 @@ import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; +import java.util.Collections; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@ -147,7 +149,10 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; +import org.apache.hadoop.net.CachedDNSToSwitchMapping; +import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.TableMapping; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.common.Storage.StorageState; @@ -160,6 +165,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.JvmPauseMonitor; @@ -319,6 +325,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private final SecretKeyManagerService secretKeyManagerService; private Clock systemClock; + private DNSToSwitchMapping dnsToSwitchMapping; /** * Creates a new StorageContainerManager. Configuration will be @@ -703,11 +710,22 @@ private void initializeSystemManagers(OzoneConfiguration conf, .build(); } + Class dnsToSwitchMappingClass = + conf.getClass( + DFSConfigKeysLegacy.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + TableMapping.class, DNSToSwitchMapping.class); + DNSToSwitchMapping newInstance = ReflectionUtils.newInstance( + dnsToSwitchMappingClass, conf); + dnsToSwitchMapping = + ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance + : new CachedDNSToSwitchMapping(newInstance)); + if (configurator.getScmNodeManager() != null) { scmNodeManager = configurator.getScmNodeManager(); } else { scmNodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, - clusterMap, scmContext, scmLayoutVersionManager); + clusterMap, scmContext, scmLayoutVersionManager, + this::resolveNodeLocation); } placementMetrics = SCMContainerPlacementMetrics.create(); @@ -2190,4 +2208,18 @@ public void scmHAMetricsUpdate(String leaderId) { public ReconfigurationHandler getReconfigurationHandler() { return reconfigurationHandler; } + + public String resolveNodeLocation(String hostname) { + List hosts = Collections.singletonList(hostname); + List resolvedHosts = dnsToSwitchMapping.resolve(hosts); + if (resolvedHosts != null && !resolvedHosts.isEmpty()) { + String location = resolvedHosts.get(0); + LOG.debug("Node {} resolved to location {}", hostname, location); + return location; + } else { + LOG.debug("Node resolution did not yield any result for {}", hostname); + return null; + } + } + } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index 4ee186323ea..af905f4db52 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.server; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -43,6 +44,7 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -61,6 +63,11 @@ public class TestSCMBlockProtocolServer { private ScmBlockLocationProtocolServerSideTranslatorPB service; private static final int NODE_COUNT = 10; + private static final Map EDGE_NODES = ImmutableMap.of( + "edge0", "/rack0", + "edge1", "/rack1" + ); + @BeforeEach void setUp(@TempDir File dir) throws Exception { config = SCMTestUtils.getConf(dir); @@ -75,6 +82,7 @@ void setUp(@TempDir File dir) throws Exception { nodeMapping.add(dn.getIpAddress() + "=" + rack); datanodes.add(dn); } + EDGE_NODES.forEach((n, r) -> nodeMapping.add(n + "=" + r)); config.set(StaticMapping.KEY_HADOOP_CONFIGURED_NODE_MAPPING, String.join(",", nodeMapping)); @@ -113,18 +121,31 @@ void sortDatanodesRelativeToDatanode() { Assertions.assertEquals(dn, sorted.get(0), "Source node should be sorted very first"); - for (int i = 1; i < NODE_COUNT / 2; i++) { - DatanodeDetails item = sorted.get(i); - Assertions.assertEquals(dn.getNetworkLocation(), - item.getNetworkLocation(), - "Nodes in the same rack should be sorted first"); - } - for (int i = NODE_COUNT / 2; i < NODE_COUNT; i++) { - DatanodeDetails item = sorted.get(i); - Assertions.assertNotEquals(dn.getNetworkLocation(), - item.getNetworkLocation(), - "Nodes in the other rack should be sorted last"); - } + assertRackOrder(dn.getNetworkLocation(), sorted); + } + } + + @Test + void sortDatanodesRelativeToNonDatanode() { + List datanodes = getNetworkNames(); + + for (Map.Entry entry : EDGE_NODES.entrySet()) { + assertRackOrder(entry.getValue(), + server.sortDatanodes(datanodes, entry.getKey())); + } + } + + private static void assertRackOrder(String rack, List list) { + int size = list.size(); + + for (int i = 0; i < size / 2; i++) { + Assertions.assertEquals(rack, list.get(i).getNetworkLocation(), + "Nodes in the same rack should be sorted first"); + } + + for (int i = size / 2; i < size; i++) { + Assertions.assertNotEquals(rack, list.get(i).getNetworkLocation(), + "Nodes in the other rack should be sorted last"); } } From 7f7ef6790055464b3091d7ad9df603c63a180103 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:51:23 +0100 Subject: [PATCH 16/51] HDDS-9789. Bump commons-cli to 1.6.0 (#5681) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4ebfae8215f..9cfbee8d2d0 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 4 1.9.4 - 1.2 + 1.6.0 1.15 3.2.2 1.21 From 289739fccdccd23829a0b2692f53a2bb66d50a6a Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Tue, 28 Nov 2023 21:32:36 +0200 Subject: [PATCH 17/51] HDDS-9266. Acceptance test for snapshot data validation after OM bootstrapping (#5350) --- .../docker-config-ratis-om-bootstrap | 20 +++ .../compose/ozonesecure-ha/om-bootstrap.yaml | 63 +++++++++ .../ozonesecure-ha/test-om-bootstrap.sh | 121 ++++++++++++++++++ .../dist/src/main/smoketest/commonlib.robot | 2 +- .../data-creation-before-om-bootstrap.robot | 96 ++++++++++++++ .../data-validation-after-om-bootstrap.robot | 99 ++++++++++++++ 6 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/om-bootstrap.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-om-bootstrap.sh create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/data-creation-before-om-bootstrap.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/data-validation-after-om-bootstrap.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap new file mode 100644 index 00000000000..d91ceb00418 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.ratis.log.purge.gap=50 +OZONE-SITE.XML_ozone.om.ratis.segment.size=16KB +OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size=16KB +OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold=500 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/om-bootstrap.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/om-bootstrap.yaml new file mode 100644 index 00000000000..90a38202140 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/om-bootstrap.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +x-OM-Ratis-config: + &common-env-file + env_file: + - ./docker-config + - ./docker-config-ratis-om-bootstrap + +x-om3-setup: + &om3-setup + environment: + WAITFOR: + # Skip initialising the OM, so that the container doesn't have any data. + ENSURE_OM_INITIALIZED: + OZONE_OPTS: + # This command produces the same behavior as sleeping indefinitely. + command: [ "tail","-f","/dev/null" ] + +services: + kdc: + <<: *common-env-file + kms: + <<: *common-env-file + datanode1: + <<: *common-env-file + datanode2: + <<: *common-env-file + datanode3: + <<: *common-env-file + om1: + <<: *common-env-file + om2: + <<: *common-env-file + om3: + <<: [*om3-setup, *common-env-file] + httpfs: + <<: *common-env-file + s3g: + <<: *common-env-file + scm1.org: + <<: *common-env-file + scm2.org: + <<: *common-env-file + scm3.org: + <<: *common-env-file + recon: + <<: *common-env-file diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-om-bootstrap.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-om-bootstrap.sh new file mode 100644 index 00000000000..2a8b7cb8b39 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-om-bootstrap.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:HA-secure + +# This test aims to validate the ozone snapshot data that have been +# installed on a bootstrapped OM after a Ratis snapshot installation. +# +# The test +# * starts the docker environment with 'om3' inactive and uninitialised +# * runs a robot test that creates keys and snapshots +# * checks that 'om3' is inactive and has no data +# * initialises 'om3' +# * starts 'om3' +# * verifies that 'om3' is running and is bootstrapping +# * runs a robot test that validates the data on 'om3' +# +# The data creation robot test +# * creates 100 metadata keys +# * creates the first snapshot +# * creates two actual keys and set the contents of each key, the same as the key name +# * creates the second snapshot +# +# The data validation robot test +# * checks that there have been checkpoints created on 'om3' +# * once checkpoints are created, the 'om3' has all the data from the leader +# * checks that 'om3' is not leader +# * transfers leadership to 'om3', so that we can perform regular leader reads +# * checks that the two snapshots exist on 'om3' +# * runs a snapshot diff between the two snapshots +# * validates that the result of the snapshot diff, contains just the two actual keys +# * does a 'key cat' on both snapshot keys and validates the contents +# * the keys are read from the snapshot and not the active file system +# * the contents of each key should be the same as the key name + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +export SECURITY_ENABLED=true +export OM_SERVICE_ID="omservice" +export SCM=scm1.org +export COMPOSE_FILE=docker-compose.yaml:om-bootstrap.yaml + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +volume="vol1" +bucket="bucket1" +snap1="snap1" +snap2="snap2" +keyPrefix="sn" +key1="key1" +key2="key2" +bootstrap_om="om3" + +execute_robot_test om1 kinit.robot + +# Data creation +execute_robot_test om1 -v VOLUME:${volume} -v BUCKET:${bucket} -v SNAP_1:${snap1} -v SNAP_2:${snap2} -v KEY_PREFIX:${keyPrefix} -v KEY_1:${key1} -v KEY_2:${key2} omha/data-creation-before-om-bootstrap.robot + +echo "Check that om3 isn't running" +om3_service=$(execute_command_in_container om3 ps aux | grep 'OzoneManagerStarter' || true) + +if [[ $om3_service != "" ]] +then + echo "om3 is running, exiting..." + exit 1 +fi + +echo "Check that om3 has no data" +om3_data=$(execute_command_in_container om3 ls -lah /data | grep 'metadata' || true) + +if [[ $om3_data != "" ]] +then + echo "om3 has data, exiting..." + exit 1 +fi + +# Init om3 and start the om daemon in the background +execute_command_in_container om3 ozone om --init +execute_command_in_container -d om3 ozone om +wait_for_port om3 9872 120 + +echo "Check that om3 is running" +om3_service=$(execute_command_in_container om3 ps aux | grep 'OzoneManagerStarter' || true) + +if [[ $om3_service == "" ]] +then + echo "om3 isn't running, exiting..." + exit 1 +fi + +echo "Check that om3 has data" +om3_data=$(execute_command_in_container om3 ls -lah /data | grep 'metadata' || true) + +if [[ $om3_data == "" ]] +then + echo "om3 has no data, exiting..." + exit 1 +fi + +execute_robot_test om3 kinit.robot + +# This test checks the disk on the node it's running. It needs to be run on om3. +execute_robot_test om3 -v BOOTSTRAPPED_OM:${bootstrap_om} -v VOLUME:${volume} -v BUCKET:${bucket} -v SNAP_1:${snap1} -v SNAP_2:${snap2} -v KEY_PREFIX:${keyPrefix} -v KEY_1:${key1} -v KEY_2:${key2} omha/data-validation-after-om-bootstrap.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot index 8f142027f30..7d9edcdef44 100644 --- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot @@ -28,7 +28,7 @@ ${OM_SERVICE_ID} om *** Keywords *** Get test user principal [arguments] ${user} - ${instance} = Execute hostname | sed 's/scm[0-9].org/scm/' | sed 's/om[0-9]/om/' + ${instance} = Execute hostname | sed 's/scm[0-9].org/scm/;s/scm[0-9]/scm/;s/om[0-9]/om/' [return] ${user}/${instance}@EXAMPLE.COM Kinit HTTP user diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/data-creation-before-om-bootstrap.robot b/hadoop-ozone/dist/src/main/smoketest/omha/data-creation-before-om-bootstrap.robot new file mode 100644 index 00000000000..4d1100aee57 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/omha/data-creation-before-om-bootstrap.robot @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test for creating data needed for om bootstrap load test. +Resource ../commonlib.robot +Test Timeout 5 minutes +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Variables *** +${TMP_FILE} tmp.txt +${VOLUME} +${BUCKET} +${SNAP_1} +${SNAP_2} +${KEY_PREFIX} +${KEY_1} +${KEY_2} + +*** Keywords *** +Create volume and bucket + [arguments] ${volume} ${bucket} + ${vol_res} = Execute ozone sh volume create /${volume} + Should Be Empty ${vol_res} + ${bucket_res} = Execute ozone sh bucket create /${volume}/${bucket} + Should Be Empty ${bucket_res} + +Create tmp file + [arguments] ${file_name} + ${create_res} = Execute touch ${file_name} + Should Be Empty ${create_res} + ${ls_grep_res} = Execute ls -lah | grep '${file_name}' + Should contain ${ls_grep_res} ${file_name} + +Delete tmp file + [arguments] ${file_name} + Execute rm ${file_name} + ${file_exists} = Execute [[ -f ${file_name} ]] && echo "File exists" || echo "File doesn't exist" + Should contain ${file_exists} File doesn't exist + +Create a key and set contents same as the keyName + [arguments] ${volume} ${bucket} ${key_prefix} ${key_name} ${tmp_file} + Execute echo "${key_prefix}/${key_name}" > ${tmp_file} + ${key_res} = Execute ozone sh key put /${volume}/${bucket}/${key_prefix}/${key_name} ${tmp_file} + Should Be Empty ${key_res} + ${key_cat_res} = Execute ozone sh key cat /${volume}/${bucket}/${key_prefix}/${key_name} + Should contain ${key_cat_res} ${key_prefix}/${key_name} + +Create actual keys + [arguments] ${volume} ${bucket} ${key_prefix} ${key_1} ${key2} ${tmp_file} + Create a key and set contents same as the keyName ${volume} ${bucket} ${key_prefix} ${key_1} ${tmp_file} + Create a key and set contents same as the keyName ${volume} ${bucket} ${key_prefix} ${key_2} ${tmp_file} + +Create metadata keys + [arguments] ${threads} ${key_num} ${volume} ${bucket} + ${freon_res} = Execute ozone freon omkg -t ${threads} -n ${key_num} -v ${volume} -b ${bucket} + Should contain ${freon_res} Successful executions: ${key_num} + +Create snapshot + [arguments] ${volume} ${bucket} ${snapshot} + ${snap_res} = Execute ozone sh snapshot create /${volume}/${bucket} ${snapshot} + Should Be Empty ${snap_res} + +*** Test Cases *** +Volume-bucket init + Create volume and bucket ${VOLUME} ${BUCKET} + +Create 100 metadata keys under /${VOLUME}/${BUCKET} + Create metadata keys 10 100 ${VOLUME} ${BUCKET} + +Create snapshot '${SNAP_1}' + Create snapshot ${VOLUME} ${BUCKET} ${SNAP_1} + +Create tmp file to be used for key creation + Create tmp file ${TMP_FILE} + +Create 2 actual keys with prefix '${KEY_PREFIX}', key contents the same as the key name + Create actual keys ${VOLUME} ${BUCKET} ${KEY_PREFIX} ${KEY_1} ${KEY_2} ${TMP_FILE} + +Create snapshot '${SNAP_2}' + Create snapshot ${VOLUME} ${BUCKET} ${SNAP_2} + +Cleanup tmp file + Delete tmp file ${TMP_FILE} diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/data-validation-after-om-bootstrap.robot b/hadoop-ozone/dist/src/main/smoketest/omha/data-validation-after-om-bootstrap.robot new file mode 100644 index 00000000000..b3ece0d3587 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/omha/data-validation-after-om-bootstrap.robot @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test for validating snapshot data after om bootstrap. +Resource ../commonlib.robot +Test Timeout 5 minutes +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Variables *** +${BOOTSTRAPPED_OM} +${VOLUME} +${BUCKET} +${SNAP_1} +${SNAP_2} +${KEY_PREFIX} +${KEY_1} +${KEY_2} + +*** Keywords *** +Number of checkpoints equals 2 + ${checkpoints} = Execute ls -lah /data/metadata/db.snapshots/checkpointState | grep 'om.db-' | wc -l + Should be true ${checkpoints} == 2 + +Check current leader is different than OM + [arguments] ${om} + ${leader} Execute ozone admin om roles -id=omservice | grep 'LEADER' | awk -F ':' '{ print $1 }' + Should not contain ${leader} ${om} + +Transfer leadership to OM + [arguments] ${new_leader} + ${result} = Execute ozone admin om transfer --service-id=omservice -n ${new_leader} + Should Contain ${result} Transfer leadership successfully + +Check snapshots on OM + [arguments] ${volume} ${bucket} ${snap_1} ${snap_2} + ${snap_list} Execute ozone sh snapshot list /${volume}/${bucket} + Should not contain ${snap_list} [ ] + ${snap1_res} = Execute echo "${snap_list}" | grep ${snap_1} + Should contain ${snap1_res} ${snap_1} + ${snap2_res} = Execute echo "${snap_list}" | grep ${snap_2} + Should contain ${snap2_res} ${snap_2} + +Run snap diff and get response + [arguments] ${volume} ${bucket} ${snap_1} ${snap_2} + ${diff_res} = Execute ozone sh snapshot diff /${volume}/${bucket} ${snap_1} ${snap_2} + [return] ${diff_res} + +Snap diff finished + [arguments] ${volume} ${bucket} ${snap_1} ${snap_2} + ${diff_res} = Run snap diff and get response ${volume} ${bucket} ${snap_1} ${snap_2} + Should contain ${diff_res} Difference between snapshot: ${snap_1} and snapshot: ${snap_2} + +Run snap diff and validate results + [arguments] ${volume} ${bucket} ${snap_1} ${snap_2} ${key_1} ${key_2} + Wait Until Keyword Succeeds 2min 3sec Snap diff finished ${volume} ${bucket} ${snap_1} ${snap_2} + ${diff_res} = Run snap diff and get response ${volume} ${bucket} ${snap_1} ${snap_2} + ${key_num} = Execute echo "${diff_res}" | grep 'key' | wc -l + Should be true ${key_num} == 2 + ${diff_key1} = Execute echo "${diff_res}" | grep ${key_1} | wc -l + Should be true ${diff_key1} == 1 + ${diff_key2} = Execute echo "${diff_res}" | grep ${key_2} | wc -l + Should be true ${diff_key2} == 1 + +Validate keys under snapshot + [arguments] ${volume} ${bucket} ${snap} ${key_prefix} ${key_1} ${key_2} + ${key1_res} = Execute ozone sh key cat /${volume}/${bucket}/.snapshot/${snap}/${key_prefix}/${key_1} + Should contain ${key1_res} ${key_prefix}/${key_1} + ${key2_res} = Execute ozone sh key cat /${volume}/${bucket}/.snapshot/${snap}/${key_prefix}/${key_2} + Should contain ${key2_res} ${key_prefix}/${key_2} + +*** Test Cases *** +Check number of checkpoints made + Wait Until Keyword Succeeds 3min 5sec Number of checkpoints equals 2 + +Check current leader and transfer leadership to '${BOOTSTRAPPED_OM}' + Check current leader is different than OM ${BOOTSTRAPPED_OM} + Transfer leadership to OM ${BOOTSTRAPPED_OM} + +Snapshots exist on '${BOOTSTRAPPED_OM}' + Check snapshots on OM ${VOLUME} ${BUCKET} ${SNAP_1} ${SNAP_2} + +Run snap diff on '${BOOTSTRAPPED_OM}' and check diff keys + Run snap diff and validate results ${VOLUME} ${BUCKET} ${SNAP_1} ${SNAP_2} ${KEY_1} ${KEY_2} + +Cat snapshot keys and validate contents + Validate keys under snapshot ${VOLUME} ${BUCKET} ${SNAP_2} ${KEY_PREFIX} ${KEY_1} ${KEY_2} From 07f31082eefcf38417cd60322511bd94d493a5b0 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran <47532440+swamirishi@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:40:00 +0530 Subject: [PATCH 18/51] HDDS-9778. Disable rocksDB cache for snapshot (#5676) --- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 11 +++++------ .../recon/recovery/ReconOmMetadataManagerImpl.java | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 6685e9a2ec8..42179294e56 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -380,7 +380,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) omEpoch = 0; setStore(loadDB(conf, dir, name, true, java.util.Optional.of(Boolean.TRUE), Optional.empty())); - initializeOmTables(false); + initializeOmTables(CacheType.PARTIAL_CACHE, false); } @@ -413,7 +413,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) setStore(loadDB(conf, metaDir, dbName, false, java.util.Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false)); - initializeOmTables(false); + initializeOmTables(CacheType.PARTIAL_CACHE, false); } catch (IOException e) { stop(); throw e; @@ -557,7 +557,7 @@ public void start(OzoneConfiguration configuration) throws IOException { this.store = loadDB(configuration, metaDir); - initializeOmTables(true); + initializeOmTables(CacheType.FULL_CACHE, true); } snapshotChainManager = new SnapshotChainManager(this); @@ -652,15 +652,14 @@ public static DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { * * @throws IOException */ - protected void initializeOmTables(boolean addCacheMetrics) + protected void initializeOmTables(CacheType cacheType, + boolean addCacheMetrics) throws IOException { userTable = this.store.getTable(USER_TABLE, String.class, PersistedUserVolumeInfo.class); checkTableStatus(userTable, USER_TABLE, addCacheMetrics); - CacheType cacheType = CacheType.FULL_CACHE; - volumeTable = this.store.getTable(VOLUME_TABLE, String.class, OmVolumeArgs.class, cacheType); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java index 856f7cb083a..ad0526363df 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.hdds.utils.db.cache.TableCache; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; @@ -109,7 +110,7 @@ private void initializeNewRdbStore(File dbFile) throws IOException { LOG.error("Unable to initialize Recon OM DB snapshot store.", ioEx); } if (getStore() != null) { - initializeOmTables(true); + initializeOmTables(TableCache.CacheType.FULL_CACHE, true); omTablesInitialized = true; } } From 799b20a08bf2e9746db2984da0cfcceabccce228 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Wed, 29 Nov 2023 09:03:14 +0100 Subject: [PATCH 19/51] HDDS-5506. Use secure cluster for upgrade acceptance tests (#5585) --- .../src/main/compose/common/security.conf | 105 ++++++++++++++++++ .../src/main/compose/upgrade/compose/ha/.env | 2 + .../upgrade/compose/ha/docker-compose.yaml | 76 ++++++++++++- .../compose/upgrade/compose/ha/docker-config | 6 +- .../main/compose/upgrade/compose/ha/krb5.conf | 41 +++++++ .../main/compose/upgrade/compose/ha/load.sh | 3 +- .../main/smoketest/ec/upgrade-ec-check.robot | 3 +- .../src/main/smoketest/s3/commonawslib.robot | 8 +- .../snapshot/upgrade-snapshot-check.robot | 1 + .../src/main/smoketest/upgrade/generate.robot | 6 +- .../src/main/smoketest/upgrade/validate.robot | 7 +- 11 files changed, 241 insertions(+), 17 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/common/security.conf create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/krb5.conf diff --git a/hadoop-ozone/dist/src/main/compose/common/security.conf b/hadoop-ozone/dist/src/main/compose/common/security.conf new file mode 100644 index 00000000000..7b74224e603 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/common/security.conf @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For HttpFS service it is required to enable proxying users. +CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=* +CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=* + +CORE-SITE.XML_dfs.data.transfer.protection=authentication +CORE-SITE.XML_hadoop.security.authentication=kerberos +CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT" +CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms + +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab +OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM + +OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM + +OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM + +HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +OZONE-SITE.XML_hdds.block.token.enabled=true +OZONE-SITE.XML_hdds.container.token.enabled=true +OZONE-SITE.XML_hdds.grpc.tls.enabled=true +OZONE-SITE.XML_ozone.security.enabled=true +OZONE-SITE.XML_ozone.acl.enabled=true +OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer +OZONE-SITE.XML_ozone.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" +OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true +OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true +OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer + +OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m +OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m +OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h + +OZONE-SITE.XML_ozone.om.http.auth.type=kerberos +OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos +OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos +OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos +OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos + +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/db@EXAMPLE.COM +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab + +CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false +CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret +CORE-SITE.XML_hadoop.http.authentication.type=kerberos +CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +CORE-SITE.XML_hadoop.security.authorization=true +HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* + +HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos +HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPLE.COM +HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos +HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* + +OZONE_DATANODE_SECURE_USER=root diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env index 4d1c35c3b2d..4e01ec92416 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env @@ -14,10 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +HADOOP_VERSION=${hadoop.version} HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version} +OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} OZONE_DIR=/opt/hadoop OZONE_VOLUME=./data OM_SERVICE_ID=omservice diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index f7fea2f5620..186228fe60f 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -21,6 +21,7 @@ x-common-config: &common-config env_file: - docker-config + - ../../../common/security.conf image: ${OZONE_IMAGE} x-environment: @@ -29,6 +30,7 @@ x-environment: OZONE_UPGRADE_TO: ${OZONE_UPGRADE_TO:-0} OZONE_UPGRADE_FROM: ${OZONE_UPGRADE_FROM:-0} OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + WAITFOR: kdc:88 x-datanode: &datanode @@ -59,35 +61,75 @@ x-om: - 9872 x-volumes: + - &keytabs ../../../_keytabs:/etc/security/keytabs + - &krb5conf ./krb5.conf:/etc/krb5.conf - &ozone-dir ../../../..:${OZONE_DIR} - &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py services: + kdc: + command: ["krb5kdc","-n"] + hostname: kdc + image: ${OZONE_TESTKRB5_IMAGE} + networks: + net: + ipv4_address: 10.9.0.2 + volumes: + - *keytabs + - ../../../..:/opt/hadoop + kms: + command: ["hadoop", "kms"] + hostname: kms + env_file: + - docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + image: apache/hadoop:${HADOOP_VERSION} + networks: + net: + ipv4_address: 10.9.0.3 + ports: + - 9600:9600 + volumes: + - ${OZONE_VOLUME}/kms:/data + - *keytabs + - *krb5conf + - ../../../..:/opt/ozone + - *transformation om1: <<: *om + hostname: om1 networks: net: ipv4_address: 10.9.0.11 volumes: - ${OZONE_VOLUME}/om1:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation om2: <<: *om + hostname: om2 networks: net: ipv4_address: 10.9.0.12 volumes: - ${OZONE_VOLUME}/om2:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation om3: <<: *om + hostname: om3 networks: net: ipv4_address: 10.9.0.13 volumes: - ${OZONE_VOLUME}/om3:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation @@ -96,83 +138,107 @@ services: environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION <<: *environment + hostname: scm1.org networks: net: ipv4_address: 10.9.0.14 volumes: - ${OZONE_VOLUME}/scm1:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation scm2: <<: *scm environment: - WAITFOR: scm1:9894 + WAITFOR: scm1.org:9894 ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION <<: *environment + hostname: scm2.org networks: net: ipv4_address: 10.9.0.15 volumes: - ${OZONE_VOLUME}/scm2:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation scm3: <<: *scm environment: - WAITFOR: scm2:9894 + WAITFOR: scm2.org:9894 ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION <<: *environment + hostname: scm3.org networks: net: ipv4_address: 10.9.0.16 volumes: - ${OZONE_VOLUME}/scm3:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation dn1: <<: *datanode + hostname: dn1 networks: net: ipv4_address: 10.9.0.17 volumes: - ${OZONE_VOLUME}/dn1:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation dn2: <<: *datanode + hostname: dn2 networks: net: ipv4_address: 10.9.0.18 volumes: - ${OZONE_VOLUME}/dn2:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation dn3: <<: *datanode + hostname: dn3 networks: net: ipv4_address: 10.9.0.19 volumes: - ${OZONE_VOLUME}/dn3:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation dn4: <<: *datanode + hostname: dn4 networks: net: ipv4_address: 10.9.0.20 volumes: - ${OZONE_VOLUME}/dn4:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation dn5: <<: *datanode + hostname: dn5 networks: net: ipv4_address: 10.9.0.21 volumes: - ${OZONE_VOLUME}/dn5:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation recon: @@ -180,6 +246,7 @@ services: <<: *common-config environment: <<: *environment + hostname: recon networks: net: ipv4_address: 10.9.0.22 @@ -187,6 +254,8 @@ services: - 9888:9888 volumes: - ${OZONE_VOLUME}/recon:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation s3g: @@ -194,6 +263,7 @@ services: <<: *common-config environment: <<: *environment + hostname: s3g networks: net: ipv4_address: 10.9.0.23 @@ -201,6 +271,8 @@ services: - 9878:9878 volumes: - ${OZONE_VOLUME}/s3g:/data + - *keytabs + - *krb5conf - *ozone-dir - *transformation diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config index a022f37c9a5..e252ebecaf1 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config @@ -27,9 +27,9 @@ OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.service.ids=scmservice OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 +OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org +OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org +OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org OZONE-SITE.XML_ozone.scm.ratis.enable=true OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/krb5.conf b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/krb5.conf new file mode 100644 index 00000000000..eefc5b9c685 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/krb5.conf @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[logging] +default = FILE:/var/log/krb5libs.log +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + dns_canonicalize_hostname = false + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = EXAMPLE.COM + +[realms] + EXAMPLE.COM = { + kdc = kdc + admin_server = kdc + max_renewable_life = 7d + } + +[domain_realm] + .example.com = EXAMPLE.COM + example.com = EXAMPLE.COM + diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh index 59ec12f4c66..6b4241b2890 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh @@ -25,6 +25,7 @@ source "$TEST_DIR/testlib.sh" export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml" export OM_SERVICE_ID=omservice -create_data_dirs "${OZONE_VOLUME}"/{om1,om2,om3,dn1,dn2,dn3,dn4,dn5,recon,s3g,scm1,scm2,scm3} +export SECURITY_ENABLED="true" +create_data_dirs "${OZONE_VOLUME}"/{om1,om2,om3,dn1,dn2,dn3,dn4,dn5,kms,recon,s3g,scm1,scm2,scm3} echo "Using docker cluster defined in $COMPOSE_FILE" diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot b/hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot index dbfd9e81eb5..b365960ba57 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot @@ -17,6 +17,7 @@ Documentation Test EC during upgrade Library OperatingSystem Resource lib.resource +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab *** Test Cases *** Test EC Prior To Finalization @@ -42,4 +43,4 @@ Test EC After Finalization Verify Bucket EC Replication Config /ectest-new/testpropchange RS 3 2 1048576 Execute ozone sh key put -r rs-3-2-1024k -t EC /ectest-new/ectest/core-site.xml /etc/hadoop/core-site.xml Key Should Match Local File /ectest-new/ectest/core-site.xml /etc/hadoop/core-site.xml - Verify Key EC Replication Config /ectest-new/ectest/core-site.xml RS 3 2 1048576 \ No newline at end of file + Verify Key EC Replication Config /ectest-new/ectest/core-site.xml RS 3 2 1048576 diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index ae57bf82a80..c0b2c9f7bfa 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -80,8 +80,12 @@ Setup v4 headers Setup secure v4 headers ${result} = Execute and Ignore error ozone s3 getsecret ${OM_HA_PARAM} - ${output} = Run Keyword And Return Status Should Contain ${result} S3_SECRET_ALREADY_EXISTS - Return From Keyword if ${output} + ${exists} = Run Keyword And Return Status Should Contain ${result} S3_SECRET_ALREADY_EXISTS + IF ${exists} + Execute ozone s3 revokesecret -y ${OM_HA_PARAM} + ${result} = Execute ozone s3 getsecret ${OM_HA_PARAM} + END + ${accessKey} = Get Regexp Matches ${result} (?<=awsAccessKey=).* # Use a valid user that are created in the Docket image Ex: testuser if it is not a secure cluster ${accessKey} = Get Variable Value ${accessKey} testuser diff --git a/hadoop-ozone/dist/src/main/smoketest/snapshot/upgrade-snapshot-check.robot b/hadoop-ozone/dist/src/main/smoketest/snapshot/upgrade-snapshot-check.robot index 33916547448..6003e6dbb33 100644 --- a/hadoop-ozone/dist/src/main/smoketest/snapshot/upgrade-snapshot-check.robot +++ b/hadoop-ozone/dist/src/main/smoketest/snapshot/upgrade-snapshot-check.robot @@ -19,6 +19,7 @@ Library OperatingSystem Library BuiltIn Resource ../commonlib.robot Default Tags pre-finalized-snapshot-tests +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 5 minutes *** Variables *** diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot index 2bfde82b043..3730e265868 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot @@ -19,6 +19,7 @@ Library OperatingSystem Library BuiltIn Resource ../commonlib.robot Resource ../s3/commonawslib.robot +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 5 minutes *** Variables *** @@ -49,11 +50,8 @@ Create key in the bucket in s3v volume Should not contain ${output} Failed Execute and checkrc rm /tmp/sourcekey 0 -Setup credentials for S3 - # TODO: Run "Setup secure v4 headers" instead when security is enabled - Run Keyword Setup dummy credentials for S3 - Try to create a bucket using S3 API + [setup] Setup v4 headers # Note: S3 API returns error if the bucket already exists ${random} = Generate Ozone String ${output} = Create bucket with name ${PREFIX}-bucket-${random} diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot index 9f5b0a08bf5..0205ba40e69 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot @@ -19,6 +19,7 @@ Library OperatingSystem Library BuiltIn Resource ../commonlib.robot Resource ../s3/commonawslib.robot +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 5 minutes *** Variables *** @@ -33,11 +34,8 @@ Read data from previously created key Should contain ${output} ${PREFIX}: key created using Ozone Shell Execute and checkrc rm /tmp/key-${random} 0 -Setup credentials for S3 - # TODO: Run "Setup secure v4 headers" instead when security is enabled - Run Keyword Setup dummy credentials for S3 - Read key created with Ozone Shell using S3 API + [setup] Setup v4 headers ${output} = Execute AWSS3APICli and checkrc get-object --bucket ${PREFIX}-bucket --key key1-shell /tmp/get-result 0 Should contain ${output} "ContentLength" ${output} = Execute and checkrc cat /tmp/get-result 0 @@ -45,6 +43,7 @@ Read key created with Ozone Shell using S3 API Execute and checkrc rm /tmp/get-result 0 Read key created with S3 API using S3 API + [setup] Setup v4 headers ${output} = Execute AWSS3APICli and checkrc get-object --bucket ${PREFIX}-bucket --key key2-s3api /tmp/get-result 0 Should contain ${output} "ContentLength" ${output} = Execute and checkrc cat /tmp/get-result 0 From e84dfaa7fc2b9e33048bb15ece0e1024f5b308a8 Mon Sep 17 00:00:00 2001 From: Arafat2198 <98023601+ArafatKhan2198@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:37:22 +0530 Subject: [PATCH 20/51] HDDS-9780. Intermittent failure in testS3SecretCacheSizePostDoubleBufferFlush (#5693) --- .../ratis/TestOzoneManagerDoubleBuffer.java | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index 8d408cb5057..3f54018cda0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -340,35 +340,40 @@ public void testS3SecretCacheSizePostDoubleBufferFlush() throws IOException { Assertions.assertEquals("alice", ugiAlice.getShortUserName()); when(ozoneManager.isS3Admin(ugiAlice)).thenReturn(true); - // Create 3 secrets and store them in the cache and double buffer. - processSuccessSecretRequest(userPrincipalId1, 1, true); - processSuccessSecretRequest(userPrincipalId2, 2, true); - processSuccessSecretRequest(userPrincipalId3, 3, true); - - S3SecretCache cache = secretManager.cache(); - // Check if all the three secrets are cached. - Assertions.assertTrue(cache.get(userPrincipalId1) != null); - Assertions.assertTrue(cache.get(userPrincipalId2) != null); - Assertions.assertTrue(cache.get(userPrincipalId3) != null); - - // Flush the current buffer. - doubleBuffer.flushCurrentBuffer(); - - // Check if all the three secrets are cleared from the cache. - Assertions.assertTrue(cache.get(userPrincipalId3) == null); - Assertions.assertTrue(cache.get(userPrincipalId2) == null); - Assertions.assertTrue(cache.get(userPrincipalId1) == null); - - // cleanup metrics - doubleBuffer.stopDaemon(); - OzoneManagerDoubleBufferMetrics metrics = - doubleBuffer.getOzoneManagerDoubleBufferMetrics(); - metrics.setMaxNumberOfTransactionsFlushedInOneIteration(0); - metrics.setAvgFlushTransactionsInOneIteration(0); - metrics.incrTotalSizeOfFlushedTransactions( - -metrics.getTotalNumOfFlushedTransactions()); - metrics.incrTotalNumOfFlushOperations( - -metrics.getTotalNumOfFlushOperations()); + try { + // Stop the double buffer thread to prevent automatic flushing every + // second and to enable manual flushing. + doubleBuffer.stopDaemon(); + + // Create 3 secrets and store them in the cache and double buffer. + processSuccessSecretRequest(userPrincipalId1, 1, true); + processSuccessSecretRequest(userPrincipalId2, 2, true); + processSuccessSecretRequest(userPrincipalId3, 3, true); + + S3SecretCache cache = secretManager.cache(); + // Check if all the three secrets are cached. + Assertions.assertTrue(cache.get(userPrincipalId1) != null); + Assertions.assertTrue(cache.get(userPrincipalId2) != null); + Assertions.assertTrue(cache.get(userPrincipalId3) != null); + + // Flush the current buffer. + doubleBuffer.flushCurrentBuffer(); + + // Check if all the three secrets are cleared from the cache. + Assertions.assertTrue(cache.get(userPrincipalId3) == null); + Assertions.assertTrue(cache.get(userPrincipalId2) == null); + Assertions.assertTrue(cache.get(userPrincipalId1) == null); + } finally { + // cleanup metrics + OzoneManagerDoubleBufferMetrics metrics = + doubleBuffer.getOzoneManagerDoubleBufferMetrics(); + metrics.setMaxNumberOfTransactionsFlushedInOneIteration(0); + metrics.setAvgFlushTransactionsInOneIteration(0); + metrics.incrTotalSizeOfFlushedTransactions( + -metrics.getTotalNumOfFlushedTransactions()); + metrics.incrTotalNumOfFlushOperations( + -metrics.getTotalNumOfFlushOperations()); + } } private void processSuccessSecretRequest( From 28fa30c8772d54c1927d8fd75b9508b98aec0fe3 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Wed, 29 Nov 2023 16:00:02 +0530 Subject: [PATCH 21/51] HDDS-9537. Intermittent failure in TestPipelineManagerMXBean (#5677) --- .../pipeline/TestPipelineManagerMXBean.java | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestPipelineManagerMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestPipelineManagerMXBean.java index 69233aab6e8..f1a533bdfdc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestPipelineManagerMXBean.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestPipelineManagerMXBean.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,18 +32,16 @@ import javax.management.openmbean.TabularData; import java.io.IOException; import java.lang.management.ManagementFactory; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeoutException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + /** * Test cases to verify the metrics exposed by SCMPipelineManager via MXBean. */ -@Timeout(300) +@Timeout(3000) public class TestPipelineManagerMXBean { private MiniOzoneCluster cluster; @@ -66,29 +65,35 @@ public void init() public void testPipelineInfo() throws Exception { ObjectName bean = new ObjectName( "Hadoop:service=SCMPipelineManager,name=SCMPipelineManagerInfo"); + Map pipelineStateCount = cluster + .getStorageContainerManager().getPipelineManager().getPipelineInfo(); - TabularData data = (TabularData) mbs.getAttribute(bean, "PipelineInfo"); - Map datanodeInfo = cluster.getStorageContainerManager() - .getPipelineManager().getPipelineInfo(); - verifyEquals(data, datanodeInfo); + GenericTestUtils.waitFor(() -> { + try { + final TabularData data = (TabularData) mbs.getAttribute( + bean, "PipelineInfo"); + for (Map.Entry entry : pipelineStateCount.entrySet()) { + final Integer count = entry.getValue(); + final Integer currentCount = getMetricsCount(data, entry.getKey()); + if (currentCount == null || !currentCount.equals(count)) { + return false; + } + } + return true; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 500, 3000); } - private void verifyEquals(TabularData actualData, - Map expectedData) { - assertNotNull(actualData); - assertNotNull(expectedData); - for (Object obj : actualData.values()) { - assertTrue(obj instanceof CompositeData); - CompositeData cds = (CompositeData) obj; - assertEquals(2, cds.values().size()); - Iterator it = cds.values().iterator(); - String key = it.next().toString(); - String value = it.next().toString(); - long num = Long.parseLong(value); - assertTrue(expectedData.containsKey(key)); - assertEquals(expectedData.remove(key).longValue(), num); + private Integer getMetricsCount(TabularData data, String state) { + for (Object obj : data.values()) { + CompositeData cds = assertInstanceOf(CompositeData.class, obj); + if (cds.get("key").equals(state)) { + return Integer.parseInt(cds.get("value").toString()); + } } - assertTrue(expectedData.isEmpty()); + return null; } @AfterEach From 2cbb0e634ae065c26c26d7db74c7c139034cc876 Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Wed, 29 Nov 2023 16:16:46 +0530 Subject: [PATCH 22/51] HDDS-9479. Pipeline close doesn't wait for containers to be closed. (#5604) --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 ++- .../hadoop/hdds/scm/pipeline/Pipeline.java | 16 +++++ .../hadoop/hdds/scm/node/DeadNodeHandler.java | 6 +- .../scm/node/HealthyReadOnlyNodeHandler.java | 2 +- .../hdds/scm/node/StaleNodeHandler.java | 4 +- .../scm/node/StartDatanodeAdminHandler.java | 4 +- .../scm/pipeline/PipelineActionHandler.java | 3 +- .../hdds/scm/pipeline/PipelineManager.java | 5 ++ .../scm/pipeline/PipelineManagerImpl.java | 63 ++++++++++++------- .../scm/pipeline/MockPipelineManager.java | 10 +++ .../pipeline/TestPipelineActionHandler.java | 8 +-- .../scm/pipeline/TestPipelineManagerImpl.java | 29 ++++++--- .../scm/pipeline/TestNode2PipelineMap.java | 4 +- .../hdds/scm/pipeline/TestPipelineClose.java | 8 ++- .../client/rpc/TestContainerStateMachine.java | 3 + .../TestContainerStateMachineFailures.java | 9 +-- .../TestContainerStateMachineFlushDelay.java | 4 ++ ...oneClientRetriesOnExceptionFlushDelay.java | 4 ++ .../TestOzoneClientRetriesOnExceptions.java | 4 ++ .../ozone/client/rpc/TestWatchForCommit.java | 9 +-- .../ozone/recon/TestReconAsPassiveScm.java | 3 +- .../node/TestDecommissionAndMaintenance.java | 4 ++ .../scm/pipeline/TestSCMPipelineMetrics.java | 10 +-- .../ozone/recon/scm/ReconPipelineManager.java | 3 +- 24 files changed, 158 insertions(+), 65 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 1deff3e409d..a2c00d5c214 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -420,8 +420,12 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT = "ozone.scm.pipeline.destroy.timeout"; + // We wait for 150s before closing containers + // OzoneConfigKeys#OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION. + // So, we are waiting for another 150s before deleting the pipeline + // (150 + 150) = 300s public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT = - "66s"; + "300s"; public static final String OZONE_SCM_PIPELINE_CREATION_INTERVAL = "ozone.scm.pipeline.creation.interval"; @@ -431,7 +435,7 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_PIPELINE_SCRUB_INTERVAL = "ozone.scm.pipeline.scrub.interval"; public static final String OZONE_SCM_PIPELINE_SCRUB_INTERVAL_DEFAULT = - "5m"; + "150s"; // Allow SCM to auto create factor ONE ratis pipeline. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index 42aff0678a4..486f6781f9b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@ -87,10 +87,21 @@ public static Codec getCodec() { // suggested leader id with high priority private final UUID suggestedLeaderId; + private final Instant stateEnterTime; + /** * The immutable properties of pipeline object is used in * ContainerStateManager#getMatchingContainerByPipeline to take a lock on * the container allocations for a particular pipeline. + *

+ * Since the Pipeline class is immutable, if we want to change the state of + * the Pipeline we should create a new Pipeline object with the new state. + * Make sure that you set the value of creationTimestamp properly while + * creating the new Pipeline object. + *

+ * There is no need to worry about the value of stateEnterTime as it's + * set to Instant.now when you crate the Pipeline object as part of + * state change. */ private Pipeline(PipelineID id, ReplicationConfig replicationConfig, PipelineState state, @@ -102,6 +113,7 @@ private Pipeline(PipelineID id, this.creationTimestamp = Instant.now(); this.suggestedLeaderId = suggestedLeaderId; this.replicaIndexes = new HashMap<>(); + this.stateEnterTime = Instant.now(); } /** @@ -140,6 +152,10 @@ public Instant getCreationTimestamp() { return creationTimestamp; } + public Instant getStateEnterTime() { + return stateEnterTime; + } + /** * Return the suggested leaderId which has a high priority among DNs of the * pipeline. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java index b95998c1da1..f05eb761d91 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java @@ -80,8 +80,8 @@ public void onMessage(final DatanodeDetails datanodeDetails, * action. */ LOG.info("A dead datanode is detected. {}", datanodeDetails); - destroyPipelines(datanodeDetails); closeContainers(datanodeDetails, publisher); + destroyPipelines(datanodeDetails); // Remove the container replicas associated with the dead node unless it // is IN_MAINTENANCE @@ -122,8 +122,8 @@ private void destroyPipelines(final DatanodeDetails datanodeDetails) { .ifPresent(pipelines -> pipelines.forEach(id -> { try { - pipelineManager.closePipeline( - pipelineManager.getPipeline(id), false); + pipelineManager.closePipeline(id); + pipelineManager.deletePipeline(id); } catch (PipelineNotFoundException ignore) { // Pipeline is not there in pipeline manager, // should we care? diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java index 3286133009e..2256829942f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java @@ -90,7 +90,7 @@ public void onMessage(DatanodeDetails datanodeDetails, pipelineID, pipeline.getPipelineState(), HddsProtos.NodeState.HEALTHY_READONLY, datanodeDetails.getUuidString()); - pipelineManager.closePipeline(pipeline, true); + pipelineManager.closePipeline(pipelineID); } catch (IOException ex) { LOG.error("Failed to close pipeline {} which uses HEALTHY READONLY " + "datanode {}: ", pipelineID, datanodeDetails, ex); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java index dd8cea36697..c58cd054f9f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventHandler; @@ -59,8 +58,7 @@ public void onMessage(DatanodeDetails datanodeDetails, datanodeDetails, pipelineIds); for (PipelineID pipelineID : pipelineIds) { try { - Pipeline pipeline = pipelineManager.getPipeline(pipelineID); - pipelineManager.closePipeline(pipeline, true); + pipelineManager.closePipeline(pipelineID); } catch (IOException e) { LOG.info("Could not finalize pipeline={} for dn={}", pipelineID, datanodeDetails); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java index 783eb1358e4..0951ea81144 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.node; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventHandler; @@ -57,8 +56,7 @@ public void onMessage(DatanodeDetails datanodeDetails, datanodeDetails, pipelineIds); for (PipelineID pipelineID : pipelineIds) { try { - Pipeline pipeline = pipelineManager.getPipeline(pipelineID); - pipelineManager.closePipeline(pipeline, false); + pipelineManager.closePipeline(pipelineID); } catch (IOException e) { LOG.info("Could not finalize pipeline={} for dn={}", pipelineID, datanodeDetails); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java index e33f256a447..2f1785cf174 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java @@ -84,8 +84,7 @@ private void processPipelineAction(final DatanodeDetails datanode, info.getDetailedReason()); if (action == PipelineAction.Action.CLOSE) { - pipelineManager.closePipeline( - pipelineManager.getPipeline(pid), false); + pipelineManager.closePipeline(pid); } else { LOG.error("unknown pipeline action:{}", action); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java index 2df7e6db5f9..15b0f408c56 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java @@ -115,9 +115,14 @@ NavigableSet getContainersInPipeline(PipelineID pipelineID) void openPipeline(PipelineID pipelineId) throws IOException; + @Deprecated void closePipeline(Pipeline pipeline, boolean onTimeout) throws IOException; + void closePipeline(PipelineID pipelineID) throws IOException; + + void deletePipeline(PipelineID pipelineID) throws IOException; + void closeStalePipelines(DatanodeDetails datanodeDetails); void scrubPipelines() throws IOException; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java index 56a6de3e05e..000d3e73633 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java @@ -489,17 +489,25 @@ private void closeContainersForPipeline(final PipelineID pipelineId) * put pipeline in CLOSED state. * @param pipeline - ID of the pipeline. * @param onTimeout - whether to remove pipeline after some time. - * @throws IOException + * @throws IOException throws exception in case of failure + * @deprecated Do not use this method, onTimeout is not honored. */ - @Override + @Deprecated public void closePipeline(Pipeline pipeline, boolean onTimeout) - throws IOException { - PipelineID pipelineID = pipeline.getId(); + throws IOException { + closePipeline(pipeline.getId()); + } + + /** + * Move the Pipeline to CLOSED state. + * @param pipelineID ID of the Pipeline to be closed + * @throws IOException In case of exception while closing the Pipeline + */ + public void closePipeline(PipelineID pipelineID) throws IOException { HddsProtos.PipelineID pipelineIDProtobuf = pipelineID.getProtobuf(); // close containers. closeContainersForPipeline(pipelineID); - - if (!pipeline.isClosed()) { + if (!getPipeline(pipelineID).isClosed()) { acquireWriteLock(); try { stateManager.updatePipelineState(pipelineIDProtobuf, @@ -507,15 +515,20 @@ public void closePipeline(Pipeline pipeline, boolean onTimeout) } finally { releaseWriteLock(); } - LOG.info("Pipeline {} moved to CLOSED state", pipeline); + LOG.info("Pipeline {} moved to CLOSED state", pipelineID); } metrics.removePipelineMetrics(pipelineID); - if (!onTimeout) { - // close pipeline right away. - removePipeline(pipeline); - } + } + + /** + * Deletes the Pipeline for the given PipelineID. + * @param pipelineID ID of the Pipeline to be deleted + * @throws IOException In case of exception while deleting the Pipeline + */ + public void deletePipeline(PipelineID pipelineID) throws IOException { + removePipeline(getPipeline(pipelineID)); } /** close the pipelines whose nodes' IPs are stale. @@ -535,9 +548,10 @@ public void closeStalePipelines(DatanodeDetails datanodeDetails) { pipelinesWithStaleIpOrHostname.size()); pipelinesWithStaleIpOrHostname.forEach(p -> { try { - LOG.info("Closing the stale pipeline: {}", p.getId()); - closePipeline(p, false); - LOG.info("Closed the stale pipeline: {}", p.getId()); + final PipelineID id = p.getId(); + LOG.info("Closing the stale pipeline: {}", id); + closePipeline(id); + deletePipeline(id); } catch (IOException e) { LOG.error("Closing the stale pipeline failed: {}", p, e); } @@ -568,26 +582,34 @@ public void scrubPipelines() throws IOException { ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); + long pipelineDeleteTimoutInMills = conf.getTimeDuration( + ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, + ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); List candidates = stateManager.getPipelines(); for (Pipeline p : candidates) { + final PipelineID id = p.getId(); // scrub pipelines who stay ALLOCATED for too long. if (p.getPipelineState() == Pipeline.PipelineState.ALLOCATED && (currentTime.toEpochMilli() - p.getCreationTimestamp() .toEpochMilli() >= pipelineScrubTimeoutInMills)) { + LOG.info("Scrubbing pipeline: id: {} since it stays at ALLOCATED " + - "stage for {} mins.", p.getId(), + "stage for {} mins.", id, Duration.between(currentTime, p.getCreationTimestamp()) .toMinutes()); - closePipeline(p, false); + closePipeline(id); + deletePipeline(id); } // scrub pipelines who stay CLOSED for too long. - if (p.getPipelineState() == Pipeline.PipelineState.CLOSED) { + if (p.getPipelineState() == Pipeline.PipelineState.CLOSED && + (currentTime.toEpochMilli() - p.getStateEnterTime().toEpochMilli()) + >= pipelineDeleteTimoutInMills) { LOG.info("Scrubbing pipeline: id: {} since it stays at CLOSED stage.", p.getId()); - closeContainersForPipeline(p.getId()); - removePipeline(p); + deletePipeline(id); } // If a datanode is stopped and then SCM is restarted, a pipeline can get // stuck in an open state. For Ratis, provided some other DNs that were @@ -599,8 +621,7 @@ public void scrubPipelines() throws IOException { if (isOpenWithUnregisteredNodes(p)) { LOG.info("Scrubbing pipeline: id: {} as it has unregistered nodes", p.getId()); - closeContainersForPipeline(p.getId()); - closePipeline(p, true); + closePipeline(id); } } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java index 45ab65cd3f7..6ece2ecb88f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java @@ -243,6 +243,16 @@ public void closePipeline(final Pipeline pipeline, final boolean onTimeout) HddsProtos.PipelineState.PIPELINE_CLOSED); } + @Override + public void closePipeline(PipelineID pipelineID) throws IOException { + + } + + @Override + public void deletePipeline(PipelineID pipelineID) throws IOException { + + } + @Override public void closeStalePipelines(DatanodeDetails datanodeDetails) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java index 6ac0b538630..791220f6707 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java @@ -26,10 +26,10 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; -import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.io.IOException; import java.util.UUID; /** @@ -39,12 +39,12 @@ public class TestPipelineActionHandler { @Test public void testCloseActionForMissingPipeline() - throws PipelineNotFoundException, NotLeaderException { + throws IOException { final PipelineManager manager = Mockito.mock(PipelineManager.class); final EventQueue queue = Mockito.mock(EventQueue.class); - Mockito.when(manager.getPipeline(Mockito.any(PipelineID.class))) - .thenThrow(new PipelineNotFoundException()); + Mockito.doThrow(new PipelineNotFoundException()) + .when(manager).closePipeline(Mockito.any(PipelineID.class)); final PipelineActionHandler actionHandler = new PipelineActionHandler(manager, SCMContext.emptyContext(), null); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java index 33243b650e3..48f82b5cc95 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java @@ -86,6 +86,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT; import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.ALLOCATED; import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; @@ -341,7 +342,9 @@ public void testRemovePipeline() throws Exception { } // Destroy pipeline - pipelineManager.closePipeline(pipeline, false); + pipelineManager.closePipeline(pipeline.getId()); + pipelineManager.deletePipeline(pipeline.getId()); + try { pipelineManager.getPipeline(pipeline.getId()); fail("Pipeline should not have been retrieved"); @@ -393,7 +396,8 @@ public void testPipelineReport() throws Exception { pipelineManager.getPipeline(pipeline.getId()).isOpen()); // close the pipeline - pipelineManager.closePipeline(pipeline, false); + pipelineManager.closePipeline(pipeline.getId()); + pipelineManager.deletePipeline(pipeline.getId()); // pipeline report for destroyed pipeline should be ignored nodes.subList(0, 2).forEach(dn -> sendPipelineReport(dn, pipeline, @@ -514,6 +518,8 @@ public void testScrubPipelines() throws Exception { // Allocated pipelines should not be scrubbed for 50 seconds. conf.setTimeDuration( OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, 50, TimeUnit.SECONDS); + conf.setTimeDuration( + OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 50, TimeUnit.SECONDS); PipelineManagerImpl pipelineManager = createPipelineManager(true); Pipeline allocatedPipeline = pipelineManager @@ -553,8 +559,9 @@ public void testScrubPipelines() throws Exception { .getInstance(ReplicationFactor.THREE), Pipeline.PipelineState.ALLOCATED).contains(allocatedPipeline)); - // The closedPipeline should be scrubbed, as they are scrubbed immediately - Assertions.assertFalse(pipelineManager + // The closedPipeline should not be scrubbed as the interval has not + // yet passed. + Assertions.assertTrue(pipelineManager .getPipelines(RatisReplicationConfig .getInstance(ReplicationFactor.THREE), Pipeline.PipelineState.CLOSED).contains(closedPipeline)); @@ -569,6 +576,12 @@ public void testScrubPipelines() throws Exception { .getInstance(ReplicationFactor.THREE), Pipeline.PipelineState.ALLOCATED).contains(allocatedPipeline)); + // The closedPipeline should now be scrubbed as the interval has passed + Assertions.assertFalse(pipelineManager + .getPipelines(RatisReplicationConfig + .getInstance(ReplicationFactor.THREE), + Pipeline.PipelineState.CLOSED).contains(closedPipeline)); + pipelineManager.close(); } @@ -742,11 +755,11 @@ public void testPipelineCloseFlow() throws IOException, TimeoutException { addContainer(containerInfo.getProtobuf()); //Add Container to PipelineStateMap pipelineManager.addContainerToPipeline(pipelineID, containerID); - pipelineManager.closePipeline(pipeline, false); + pipelineManager.closePipeline(pipelineID); String containerExpectedOutput = "Container " + containerID + " closed for pipeline=" + pipelineID; String pipelineExpectedOutput = - "Pipeline " + pipeline + " moved to CLOSED state"; + "Pipeline " + pipelineID + " moved to CLOSED state"; String logOutput = logCapturer.getOutput(); assertTrue(logOutput.contains(containerExpectedOutput)); assertTrue(logOutput.contains(pipelineExpectedOutput)); @@ -847,9 +860,9 @@ public void testCloseStalePipelines() throws IOException, TimeoutException { pipelineManager.closeStalePipelines(datanodeDetails); verify(pipelineManager, times(1)) - .closePipeline(stalePipelines.get(0), false); + .closePipeline(stalePipelines.get(0).getId()); verify(pipelineManager, times(1)) - .closePipeline(stalePipelines.get(1), false); + .closePipeline(stalePipelines.get(1).getId()); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java index 6604cd652f9..dd0e5d9f318 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java @@ -116,8 +116,8 @@ public void testPipelineMap() throws IOException, ratisContainer.getPipeline().getId()); Assertions.assertEquals(0, set2.size()); - pipelineManager - .closePipeline(ratisContainer.getPipeline(), false); + pipelineManager.closePipeline(ratisContainer.getPipeline().getId()); + pipelineManager.deletePipeline(ratisContainer.getPipeline().getId()); pipelines = scm.getScmNodeManager() .getPipelines(dns.get(0)); Assertions.assertFalse( diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java index bc312719cf5..a5ca909b3e7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; @@ -84,6 +85,9 @@ public class TestPipelineClose { @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build(); conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1000, TimeUnit.MILLISECONDS); @@ -136,8 +140,8 @@ public void testPipelineCloseWithClosedContainer() throws IOException, .getContainersInPipeline(ratisContainer.getPipeline().getId()); Assert.assertEquals(0, setClosed.size()); - pipelineManager - .closePipeline(ratisContainer.getPipeline(), false); + pipelineManager.closePipeline(ratisContainer.getPipeline().getId()); + pipelineManager.deletePipeline(ratisContainer.getPipeline().getId()); for (DatanodeDetails dn : ratisContainer.getPipeline().getNodes()) { // Assert that the pipeline has been removed from Node2PipelineMap as well Assert.assertFalse(scm.getScmNodeManager().getPipelines(dn) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java index ab36e079c14..7c0fcd43722 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java @@ -107,6 +107,9 @@ public void setup() throws Exception { conf.setQuietMode(false); OzoneManager.setTestSecureOmFlag(true); conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); clientConfig.setStreamBufferFlushDelay(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java index 834669bfcdf..717304a5d0a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -84,7 +85,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import org.apache.ratis.protocol.RaftGroupId; @@ -136,8 +136,9 @@ public static void init() throws Exception { conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1, - TimeUnit.SECONDS); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); RatisClientConfig ratisClientConfig = conf.getObject(RatisClientConfig.class); @@ -480,7 +481,7 @@ public void testApplyTransactionFailure() throws Exception { } // when remove pipeline, group dir including snapshot will be deleted - LambdaTestUtils.await(5000, 500, + LambdaTestUtils.await(10000, 500, () -> (!snapshot.getPath().toFile().exists())); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java index 8be227fd184..c24f209cdeb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; @@ -111,6 +112,9 @@ public void setup() throws Exception { OzoneManager.setTestSecureOmFlag(true); conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); // conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.toString()); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1) .setBlockSize(blockSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 7714f9d226e..f3a0142f488 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -100,6 +101,9 @@ public void init() throws Exception { conf.setFromObject(config); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 3); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(7) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index c84ea1c8c17..e9497cd73f5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -107,6 +108,9 @@ public void init() throws Exception { conf.setFromObject(clientConfig); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 3); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index b52b10ed53c..b9fb0d425b8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.RatisTestHelper; import org.apache.hadoop.ozone.client.ObjectStore; @@ -63,7 +64,6 @@ import org.apache.ozone.test.tag.Flaky; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import org.apache.ratis.protocol.exceptions.GroupMismatchException; import org.junit.Assert; @@ -109,9 +109,10 @@ public void init() throws Exception { OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); clientConfig.setStreamBufferFlushDelay(false); conf.setFromObject(clientConfig); - - conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 10, - TimeUnit.SECONDS); + + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java index 556923ee330..1365872e8b4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java @@ -180,7 +180,8 @@ public void testReconRestart() throws Exception { .filter(p -> !p.getId().equals(containerInfo.getPipelineID())) .findFirst(); assertTrue(pipelineToClose.isPresent()); - scmPipelineManager.closePipeline(pipelineToClose.get(), false); + scmPipelineManager.closePipeline(pipelineToClose.get().getId()); + scmPipelineManager.deletePipeline(pipelineToClose.get().getId()); // Start Recon cluster.startRecon(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index 073e0e3bbd4..2df0d09db53 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneClusterProvider; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -134,6 +135,9 @@ public static void init() { 1, SECONDS); conf.setTimeDuration(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, 0, SECONDS); + conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); ReplicationManagerConfiguration replicationConf = conf.getObject(ReplicationManagerConfiguration.class); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java index 52bd98de350..53f4ce5e16a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java @@ -86,10 +86,12 @@ public void testPipelineDestroy() { Optional pipeline = pipelineManager .getPipelines().stream().findFirst(); Assertions.assertTrue(pipeline.isPresent()); - Assertions.assertDoesNotThrow(() -> - cluster.getStorageContainerManager() - .getPipelineManager() - .closePipeline(pipeline.get(), false)); + Assertions.assertDoesNotThrow(() -> { + PipelineManager pm = cluster.getStorageContainerManager() + .getPipelineManager(); + pm.closePipeline(pipeline.get().getId()); + pm.deletePipeline(pipeline.get().getId()); + }); MetricsRecordBuilder metrics = getMetrics( SCMPipelineMetrics.class.getSimpleName()); assertCounter("NumPipelineDestroyed", 1L, metrics); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java index 0cde1c687a8..fab30cf20b2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java @@ -150,7 +150,8 @@ public void removeInvalidPipelines(List pipelinesFromScm) { } try { LOG.info("Removing invalid pipeline {} from Recon.", pipelineID); - closePipeline(p, false); + closePipeline(p.getId()); + deletePipeline(p.getId()); } catch (IOException e) { LOG.warn("Unable to remove pipeline {}", pipelineID, e); } From 827fb8180ca4383ac03720314e58befcd1b60d41 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 29 Nov 2023 11:54:58 +0000 Subject: [PATCH 23/51] HDDS-9747. Incorrect sorting order for all unhealthy replicas in RatisOverReplicationHandler (#5689) --- .../RatisOverReplicationHandler.java | 2 +- .../TestRatisOverReplicationHandler.java | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java index 7974e72bac4..a0892b28d22 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java @@ -231,7 +231,7 @@ private List sortReplicas( // prefer deleting replicas with lower sequence IDs return replicas.stream() .sorted(Comparator.comparingLong(ContainerReplica::getSequenceId) - .reversed()) + .thenComparing(ContainerReplica::hashCode)) .collect(Collectors.toList()); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java index ec4259bac6a..f69715f9b45 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -236,6 +237,46 @@ public void testOverReplicatedContainerBecomesMisReplicatedOnRemoving() getOverReplicatedHealthResult(), 0); } + @Test + public void testOverReplicatedAllUnhealthySameBCSID() + throws IOException { + Set replicas = createReplicas(container.containerID(), + ContainerReplicaProto.State.UNHEALTHY, 0, 0, 0, 0); + + ContainerReplica shouldDelete = replicas.stream() + .sorted(Comparator.comparingLong(ContainerReplica::hashCode)) + .findFirst().get(); + + Set>> commands = + testProcessing(replicas, Collections.emptyList(), + getOverReplicatedHealthResult(), 1); + Pair> commandPair + = commands.iterator().next(); + assertEquals(shouldDelete.getDatanodeDetails(), + commandPair.getKey()); + } + + @Test + public void testOverReplicatedAllUnhealthyPicksLowestBCSID() + throws IOException { + final long sequenceID = 20; + Set replicas = new HashSet<>(); + ContainerReplica lowestSequenceIDReplica = createContainerReplica( + container.containerID(), 0, IN_SERVICE, State.UNHEALTHY, sequenceID); + replicas.add(lowestSequenceIDReplica); + for (int i = 1; i < 4; i++) { + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.UNHEALTHY, sequenceID + i)); + } + Set>> commands = + testProcessing(replicas, Collections.emptyList(), + getOverReplicatedHealthResult(), 1); + Pair> commandPair + = commands.iterator().next(); + assertEquals(lowestSequenceIDReplica.getDatanodeDetails(), + commandPair.getKey()); + } + /** * Closed container with 4 closed replicas and 1 quasi closed replica. This * container is over replicated and the handler should create a delete From 6bf5374b59a37266cab12821738f44278a9b181a Mon Sep 17 00:00:00 2001 From: Raju Balpande <146973984+raju-balpande@users.noreply.github.com> Date: Wed, 29 Nov 2023 18:45:47 +0530 Subject: [PATCH 24/51] HDDS-9501. Migrate misc. tests in hadoop-ozone to JUnit5 (#5692) --- hadoop-ozone/insight/pom.xml | 8 +++---- .../ozone/insight/TestBaseInsightPoint.java | 16 ++++++------- .../insight/TestConfigurationSubCommand.java | 24 +++++++++---------- .../ozone/insight/TestLogSubcommand.java | 6 ++--- hadoop-ozone/interface-storage/pom.xml | 9 +++---- .../ozone/om/helpers/TestOmKeyInfoCodec.java | 8 +++---- .../helpers/TestOmMultipartKeyInfoCodec.java | 8 +++---- .../ozone/om/helpers/TestOmPrefixInfo.java | 22 ++++++++--------- .../om/helpers/TestOmPrefixInfoCodec.java | 6 ++--- .../helpers/TestRepeatedOmKeyInfoCodec.java | 10 ++++---- .../om/helpers/TestS3SecretValueCodec.java | 8 +++---- .../om/helpers/TestTransactionInfoCodec.java | 8 +++---- 12 files changed, 64 insertions(+), 69 deletions(-) diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml index 3524dc3c291..dc6bb58d2c2 100644 --- a/hadoop-ozone/insight/pom.xml +++ b/hadoop-ozone/insight/pom.xml @@ -27,6 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Ozone Insight Tool Apache Ozone Insight Tool jar + + false + @@ -78,11 +81,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.dropwizard.metrics metrics-core - - junit - junit - test - org.apache.ozone hdds-hadoop-dependency-test diff --git a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestBaseInsightPoint.java b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestBaseInsightPoint.java index d8943b3727b..b2d68545d06 100644 --- a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestBaseInsightPoint.java +++ b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestBaseInsightPoint.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.ozone.insight; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; @@ -42,14 +42,14 @@ public String getDescription() { Map filters = new HashMap<>(); filters.put("datanode", "123"); - Assert.assertTrue(insightPoint + Assertions.assertTrue(insightPoint .filterLog(filters, "This a log specific to [datanode=123]")); - Assert.assertFalse(insightPoint + Assertions.assertFalse(insightPoint .filterLog(filters, "This a log specific to [datanode=234]")); //with empty filters - Assert.assertTrue(insightPoint + Assertions.assertTrue(insightPoint .filterLog(new HashMap<>(), "This a log specific to [datanode=234]")); //with multiple filters @@ -57,14 +57,14 @@ public String getDescription() { filters.put("datanode", "123"); filters.put("pipeline", "abcd"); - Assert.assertFalse(insightPoint + Assertions.assertFalse(insightPoint .filterLog(filters, "This a log specific to [datanode=123]")); - Assert.assertTrue(insightPoint + Assertions.assertTrue(insightPoint .filterLog(filters, "This a log specific to [datanode=123] [pipeline=abcd]")); - Assert.assertFalse(insightPoint + Assertions.assertFalse(insightPoint .filterLog(filters, "This a log specific to [datanode=456] [pipeline=abcd]")); diff --git a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java index 0934f4ac347..9be82ebc41d 100644 --- a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java +++ b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java @@ -27,10 +27,10 @@ import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; /** * Test insight report which prints out configs. @@ -41,12 +41,12 @@ public class TestConfigurationSubCommand { private final ByteArrayOutputStream out = new ByteArrayOutputStream(); - @Before + @BeforeEach public void setup() throws Exception { System.setOut(new PrintStream(out, false, StandardCharsets.UTF_8.name())); } - @After + @AfterEach public void reset() { System.setOut(OLD_OUT); } @@ -60,12 +60,12 @@ public void testPrintConfig() throws UnsupportedEncodingException { subCommand.printConfig(CustomConfig.class, conf); final String output = out.toString(StandardCharsets.UTF_8.name()); - Assert.assertTrue(output.contains(">>> ozone.scm.client.address")); - Assert.assertTrue(output.contains("default: localhost")); - Assert.assertTrue(output.contains("current: omclient")); - Assert.assertTrue(output.contains(">>> ozone.scm.client.secure")); - Assert.assertTrue(output.contains("default: true")); - Assert.assertTrue(output.contains("current: true")); + Assertions.assertTrue(output.contains(">>> ozone.scm.client.address")); + Assertions.assertTrue(output.contains("default: localhost")); + Assertions.assertTrue(output.contains("current: omclient")); + Assertions.assertTrue(output.contains(">>> ozone.scm.client.secure")); + Assertions.assertTrue(output.contains("default: true")); + Assertions.assertTrue(output.contains("current: true")); } /** diff --git a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestLogSubcommand.java b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestLogSubcommand.java index 4bce517702b..01402085861 100644 --- a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestLogSubcommand.java +++ b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestLogSubcommand.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.ozone.insight; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; /** * Testing utility methods of the log subcommand test. @@ -36,6 +36,6 @@ public void filterLog() { + "storageLocation: \"/tmp/hadoop-neo/dfs/data\"\\n capacity: " + "250438021120\\n scmUsed: 16384\\n remaining: 212041244672\\n " + "storageType: DISK\\n failed: false\\n}\\n"); - Assert.assertEquals(10, result.split("\n").length); + Assertions.assertEquals(10, result.split("\n").length); } } diff --git a/hadoop-ozone/interface-storage/pom.xml b/hadoop-ozone/interface-storage/pom.xml index 72d8933dbe9..a8076c01109 100644 --- a/hadoop-ozone/interface-storage/pom.xml +++ b/hadoop-ozone/interface-storage/pom.xml @@ -27,6 +27,9 @@ Apache Ozone Storage Interface Apache Ozone Storage Interface jar + + false + @@ -67,12 +70,6 @@ test - - junit - junit - test - - diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java index e16ac64dc15..996c3586b43 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java @@ -29,16 +29,16 @@ import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.util.Time; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; /** diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfoCodec.java index 29e9507256f..68a49b0ce50 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfoCodec.java @@ -24,8 +24,8 @@ import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.ozone.test.GenericTestUtils; import org.apache.hadoop.util.Time; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.UUID; @@ -58,7 +58,7 @@ public void testOmMultipartKeyInfoCodec() { } catch (java.io.IOException e) { e.printStackTrace(); } - Assert.assertNotNull(data); + Assertions.assertNotNull(data); OmMultipartKeyInfo multipartKeyInfo = null; try { @@ -66,7 +66,7 @@ public void testOmMultipartKeyInfoCodec() { } catch (java.io.IOException e) { e.printStackTrace(); } - Assert.assertEquals(omMultipartKeyInfo, multipartKeyInfo); + Assertions.assertEquals(omMultipartKeyInfo, multipartKeyInfo); // When random byte data passed returns null. try { diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java index a28c6d13b57..914697b3a62 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java @@ -23,8 +23,8 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.HashMap; @@ -91,7 +91,7 @@ public void testCopyObject() { ACCESS); OmPrefixInfo clonePrefixInfo = omPrefixInfo.copyObject(); - Assert.assertEquals(omPrefixInfo, clonePrefixInfo); + Assertions.assertEquals(omPrefixInfo, clonePrefixInfo); // Change acls and check. @@ -99,7 +99,7 @@ public void testCopyObject() { IAccessAuthorizer.ACLIdentityType.USER, username, IAccessAuthorizer.ACLType.READ, ACCESS)); - Assert.assertNotEquals(omPrefixInfo, clonePrefixInfo); + Assertions.assertNotEquals(omPrefixInfo, clonePrefixInfo); } @@ -116,10 +116,10 @@ public void testgetFromProtobufOneMetadataOneAcl() { OmPrefixInfo ompri = OmPrefixInfo.getFromProtobuf(prefixInfo); - Assert.assertEquals(prefixInfoPath, ompri.getName()); - Assert.assertEquals(1, ompri.getMetadata().size()); - Assert.assertEquals(metaval, ompri.getMetadata().get(metakey)); - Assert.assertEquals(1, ompri.getAcls().size()); + Assertions.assertEquals(prefixInfoPath, ompri.getName()); + Assertions.assertEquals(1, ompri.getMetadata().size()); + Assertions.assertEquals(metaval, ompri.getMetadata().get(metakey)); + Assertions.assertEquals(1, ompri.getAcls().size()); } @Test @@ -133,8 +133,8 @@ public void testGetProtobuf() { omPrefixInfo.getMetadata().put("key", "value"); OzoneManagerStorageProtos.PersistedPrefixInfo pi = omPrefixInfo.getProtobuf(); - Assert.assertEquals(testPath, pi.getName()); - Assert.assertEquals(1, pi.getAclsCount()); - Assert.assertEquals(1, pi.getMetadataCount()); + Assertions.assertEquals(testPath, pi.getName()); + Assertions.assertEquals(1, pi.getAclsCount()); + Assertions.assertEquals(1, pi.getMetadataCount()); } } diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfoCodec.java index bf93da0150e..6015491468c 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfoCodec.java @@ -22,8 +22,8 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.LinkedList; @@ -57,6 +57,6 @@ public void testToAndFromPersistedFormat() throws IOException { OmPrefixInfo opiLoad = codec.fromPersistedFormat( codec.toPersistedFormat(opiSave)); - Assert.assertEquals("Loaded not equals to saved", opiSave, opiLoad); + Assertions.assertEquals(opiSave, opiLoad, "Loaded not equals to saved"); } } diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java index 5a9047687a6..b2602f7bb0a 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.hadoop.util.Time; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; @@ -36,10 +36,10 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; /** * Test {@link RepeatedOmKeyInfo#getCodec(boolean)}. diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3SecretValueCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3SecretValueCodec.java index a4231a30f13..19cb6861971 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3SecretValueCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3SecretValueCodec.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; /** * Test {@link S3SecretValue#getCodec()}. @@ -44,10 +44,10 @@ public void testCodecWithCorrectData() throws Exception { UUID.randomUUID().toString()); byte[] data = codec.toPersistedFormat(s3SecretValue); - Assert.assertNotNull(data); + Assertions.assertNotNull(data); S3SecretValue docdedS3Secret = codec.fromPersistedFormat(data); - Assert.assertEquals(s3SecretValue, docdedS3Secret); + Assertions.assertEquals(s3SecretValue, docdedS3Secret); } } diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestTransactionInfoCodec.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestTransactionInfoCodec.java index 7da877bc1ee..717758c0ac4 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestTransactionInfoCodec.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestTransactionInfoCodec.java @@ -21,12 +21,12 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase; import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * Test {@link TransactionInfo#getCodec()}. @@ -48,7 +48,7 @@ public void toAndFromPersistedFormat() throws Exception { TransactionInfo convertedTransactionInfo = codec.fromPersistedFormat(codec.toPersistedFormat(transactionInfo)); - Assert.assertEquals(transactionInfo, convertedTransactionInfo); + assertEquals(transactionInfo, convertedTransactionInfo); } @Test From 55836a608b24337493234ee66269d291478ed9c0 Mon Sep 17 00:00:00 2001 From: VarshaRavi <30603028+VarshaRaviCV@users.noreply.github.com> Date: Wed, 29 Nov 2023 22:38:15 +0530 Subject: [PATCH 25/51] HDDS-9497. Migrate TestRpcClient to JUnit5 (#5699) --- hadoop-ozone/client/pom.xml | 7 ++++ .../ozone/client/rpc/TestRpcClient.java | 42 +++++++------------ 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml index e3e4a0655be..fc130a4f8a5 100644 --- a/hadoop-ozone/client/pom.xml +++ b/hadoop-ozone/client/pom.xml @@ -27,6 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Apache Ozone Client Apache Ozone Client jar + + false + @@ -57,6 +60,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> junit test + + org.junit.jupiter + junit-jupiter-params + ch.qos.reload4j reload4j diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java index f7010eaef11..0f4c4769a79 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java @@ -22,22 +22,19 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.LinkedList; import java.util.List; import static org.apache.hadoop.ozone.client.rpc.RpcClient.validateOmVersion; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Run RPC Client tests. */ -@RunWith(Parameterized.class) public class TestRpcClient { private enum ValidateOmVersionTestCases { NULL_EXPECTED_NO_OM( @@ -62,10 +59,10 @@ private enum ValidateOmVersionTestCases { true ), NULL_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( - null, - OzoneManagerVersion.CURRENT, - OzoneManagerVersion.FUTURE_VERSION, - true + null, + OzoneManagerVersion.CURRENT, + OzoneManagerVersion.FUTURE_VERSION, + true ), NULL_EXPECTED_TWO_FUTURE_OM( null, @@ -193,19 +190,9 @@ private enum ValidateOmVersionTestCases { } } - @Parameterized.Parameters(name = "{0}") - public static Iterable parameters() { - return Arrays.asList(ValidateOmVersionTestCases.values()); - } - - private ValidateOmVersionTestCases testCase; - - public TestRpcClient(ValidateOmVersionTestCases testCase) { - this.testCase = testCase; - } - - @Test - public void testValidateOmVersion() { + @ParameterizedTest + @EnumSource(ValidateOmVersionTestCases.class) + public void testValidateOmVersion(ValidateOmVersionTestCases testCase) { List serviceInfoList = new LinkedList<>(); ServiceInfo.Builder b1 = new ServiceInfo.Builder(); ServiceInfo.Builder b2 = new ServiceInfo.Builder(); @@ -219,8 +206,9 @@ public void testValidateOmVersion() { b2.setOmVersion(testCase.om2Version); serviceInfoList.add(b2.build()); } - Assert.assertEquals("Running test " + testCase, testCase.validation, - validateOmVersion(testCase.expectedVersion, serviceInfoList)); + Assertions.assertEquals(testCase.validation, + validateOmVersion(testCase.expectedVersion, serviceInfoList), + "Running test " + testCase); } @Test From 6d7ba130cf5a660780aceb773bb17d738df33905 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 29 Nov 2023 09:46:03 -0800 Subject: [PATCH 26/51] HDDS-9797. Pass defaultInstance instead of a class in Proto2/3Codec. (#5695) --- .../hadoop/hdds/protocol/DatanodeDetails.java | 13 +++++++------ .../hdds/scm/container/ContainerInfo.java | 2 +- .../hadoop/hdds/scm/pipeline/Pipeline.java | 2 +- .../hadoop/hdds/utils/db/Proto2Codec.java | 19 +++++-------------- .../hadoop/hdds/utils/db/Proto3Codec.java | 19 +++++-------------- .../container/common/helpers/BlockData.java | 2 +- .../common/helpers/ChunkInfoList.java | 2 +- .../DatanodeSchemaThreeDBDefinition.java | 2 +- .../DatanodeSchemaTwoDBDefinition.java | 2 +- .../common/helpers/MoveDataNodePair.java | 2 +- .../security/x509/certificate/CertInfo.java | 2 +- .../hdds/security/x509/crl/CRLInfo.java | 2 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 ++-- .../compaction/log/CompactionLogEntry.java | 2 +- .../hdds/scm/metadata/SCMDBDefinition.java | 2 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +- .../ozone/om/helpers/OmDBAccessIdInfo.java | 2 +- .../ozone/om/helpers/OmDBTenantState.java | 13 ++++++------- .../om/helpers/OmDBUserPrincipalInfo.java | 2 +- .../ozone/om/helpers/OmDirectoryInfo.java | 13 ++++++------- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +- .../ozone/om/helpers/OmMultipartKeyInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +- .../ozone/om/helpers/RepeatedOmKeyInfo.java | 2 +- .../ozone/om/helpers/S3SecretValue.java | 11 +++++------ .../hadoop/ozone/om/helpers/SnapshotInfo.java | 3 ++- .../snapshot/SnapshotDiffReportOzone.java | 2 +- .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 2 +- .../ozone/om/OmMetadataManagerImpl.java | 2 +- .../hadoop/ozone/om/codec/OMDBDefinition.java | 13 ++++++------- .../scm/ContainerReplicaHistoryList.java | 2 +- 31 files changed, 66 insertions(+), 86 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index fe79f273846..739f6ebd656 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ExtendedDatanodeDetailsProto; import org.apache.hadoop.hdds.scm.net.NetConstants; import org.apache.hadoop.hdds.scm.net.NodeImpl; @@ -69,7 +70,7 @@ public class DatanodeDetails extends NodeImpl implements LoggerFactory.getLogger(DatanodeDetails.class); private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(HddsProtos.ExtendedDatanodeDetailsProto.class), + Proto2Codec.get(ExtendedDatanodeDetailsProto.getDefaultInstance()), DatanodeDetails::getFromProtoBuf, DatanodeDetails::getExtendedProtoBufMessage); @@ -392,7 +393,7 @@ public static DatanodeDetails getFromProtoBuf( * @return DatanodeDetails */ public static DatanodeDetails getFromProtoBuf( - HddsProtos.ExtendedDatanodeDetailsProto extendedDetailsProto) { + ExtendedDatanodeDetailsProto extendedDetailsProto) { DatanodeDetails.Builder builder; if (extendedDetailsProto.hasDatanodeDetails()) { builder = newBuilder(extendedDetailsProto.getDatanodeDetails()); @@ -480,12 +481,12 @@ public HddsProtos.DatanodeDetailsProto.Builder toProtoBuilder( /** * Returns a ExtendedDatanodeDetails protobuf message from a datanode ID. - * @return HddsProtos.ExtendedDatanodeDetailsProto + * @return ExtendedDatanodeDetailsProto */ @JsonIgnore - public HddsProtos.ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() { - HddsProtos.ExtendedDatanodeDetailsProto.Builder extendedBuilder = - HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() + public ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() { + final ExtendedDatanodeDetailsProto.Builder extendedBuilder + = ExtendedDatanodeDetailsProto.newBuilder() .setDatanodeDetails(getProtoBufMessage()); if (!Strings.isNullOrEmpty(getVersion())) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java index 8bcc3961424..b11428581e7 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java @@ -44,7 +44,7 @@ public final class ContainerInfo implements Comparable { = Comparator.comparingLong(info -> info.getLastUsed().toEpochMilli()); private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(HddsProtos.ContainerInfoProto.class), + Proto2Codec.get(HddsProtos.ContainerInfoProto.getDefaultInstance()), ContainerInfo::fromProtobuf, ContainerInfo::getProtobuf); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index 486f6781f9b..2a117de35b4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@ -62,7 +62,7 @@ public final class Pipeline { * -- the creation time may change. */ private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(HddsProtos.Pipeline.class), + Proto2Codec.get(HddsProtos.Pipeline.getDefaultInstance()), Pipeline::getFromProtobufSetCreationTimestamp, p -> p.getProtobufMessage(ClientVersion.CURRENT_VERSION), DelegatedCodec.CopyType.UNSUPPORTED); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2Codec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2Codec.java index bf3400a6e75..e6b8338d5d0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2Codec.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto2Codec.java @@ -41,25 +41,16 @@ public final class Proto2Codec /** * @return the {@link Codec} for the given class. */ - public static Codec get(Class clazz) { - final Codec codec = CODECS.computeIfAbsent(clazz, Proto2Codec::new); + public static Codec get(T t) { + final Codec codec = CODECS.computeIfAbsent(t.getClass(), + key -> new Proto2Codec<>(t)); return (Codec) codec; } - private static Parser getParser(Class clazz) { - final String name = "PARSER"; - try { - return (Parser) clazz.getField(name).get(null); - } catch (Exception e) { - throw new IllegalStateException( - "Failed to get " + name + " field from " + clazz, e); - } - } - private final Parser parser; - private Proto2Codec(Class clazz) { - this.parser = getParser(clazz); + private Proto2Codec(M m) { + this.parser = (Parser) m.getParserForType(); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto3Codec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto3Codec.java index dc0f13a7502..d353a489d9f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto3Codec.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Proto3Codec.java @@ -41,25 +41,16 @@ public final class Proto3Codec /** * @return the {@link Codec} for the given class. */ - public static Codec get(Class clazz) { - final Codec codec = CODECS.computeIfAbsent(clazz, Proto3Codec::new); + public static Codec get(T t) { + final Codec codec = CODECS.computeIfAbsent(t.getClass(), + key -> new Proto3Codec<>(t)); return (Codec) codec; } - private static Parser getParser(Class clazz) { - final String name = "PARSER"; - try { - return (Parser) clazz.getField(name).get(null); - } catch (Exception e) { - throw new IllegalStateException( - "Failed to get " + name + " field from " + clazz, e); - } - } - private final Parser parser; - private Proto3Codec(Class clazz) { - this.parser = getParser(clazz); + private Proto3Codec(M m) { + this.parser = (Parser) m.getParserForType(); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java index c38fb88ee98..8f89be3c118 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java @@ -36,7 +36,7 @@ */ public class BlockData { private static final Codec CODEC = new DelegatedCodec<>( - Proto3Codec.get(ContainerProtos.BlockData.class), + Proto3Codec.get(ContainerProtos.BlockData.getDefaultInstance()), BlockData::getFromProtoBuf, BlockData::getProtoBufMessage); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java index 0254dfe6bd7..6f31ee40c4b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java @@ -33,7 +33,7 @@ */ public class ChunkInfoList { private static final Codec CODEC = new DelegatedCodec<>( - Proto3Codec.get(ContainerProtos.ChunkInfoList.class), + Proto3Codec.get(ContainerProtos.ChunkInfoList.getDefaultInstance()), ChunkInfoList::getFromProtoBuf, ChunkInfoList::getProtoBufMessage, DelegatedCodec.CopyType.SHALLOW); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java index 745e1153dac..1d1c7faa69b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java @@ -90,7 +90,7 @@ public class DatanodeSchemaThreeDBDefinition String.class, FixedLengthStringCodec.get(), DeletedBlocksTransaction.class, - Proto2Codec.get(DeletedBlocksTransaction.class)); + Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); private static String separator = ""; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java index 50e181147e5..cc78aa6a92c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java @@ -74,7 +74,7 @@ public class DatanodeSchemaTwoDBDefinition Long.class, LongCodec.get(), StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction.class, - Proto2Codec.get(DeletedBlocksTransaction.class)); + Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); public DatanodeSchemaTwoDBDefinition(String dbPath, ConfigurationSource config) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java index 20de146813b..42e8f8202cb 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java @@ -33,7 +33,7 @@ */ public class MoveDataNodePair { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(MoveDataNodePairProto.class), + Proto2Codec.get(MoveDataNodePairProto.getDefaultInstance()), MoveDataNodePair::getFromProtobuf, pair -> pair.getProtobufMessage(ClientVersion.CURRENT_VERSION), DelegatedCodec.CopyType.SHALLOW); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/CertInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/CertInfo.java index 88854f33023..3a1df1bd865 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/CertInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/CertInfo.java @@ -37,7 +37,7 @@ */ public final class CertInfo implements Comparable, Serializable { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(CertInfoProto.class), + Proto2Codec.get(CertInfoProto.getDefaultInstance()), CertInfo::fromProtobuf, CertInfo::getProtobuf); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java index 52545983c51..02a9d12ebda 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java @@ -41,7 +41,7 @@ public final class CRLInfo implements Comparator, Comparable { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(CRLInfoProto.class), + Proto2Codec.get(CRLInfoProto.getDefaultInstance()), proto -> fromProtobuf(proto, CRLCodec::toIOException), CRLInfo::getProtobuf); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 1a2bc94a470..fe495e7b061 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -255,8 +255,8 @@ public DBStoreBuilder addCodec(Class type, Codec codec) { return this; } - public DBStoreBuilder addProto2Codec(Class type) { - return addCodec(type, Proto2Codec.get(type)); + public DBStoreBuilder addProto2Codec(T type) { + return addCodec((Class)type.getClass(), Proto2Codec.get(type)); } public DBStoreBuilder setDBOptions(ManagedDBOptions option) { diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index 557aa9e3999..bbd24d1fc32 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -36,7 +36,7 @@ public final class CompactionLogEntry implements CopyObject { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(CompactionLogEntryProto.class), + Proto2Codec.get(CompactionLogEntryProto.getDefaultInstance()), CompactionLogEntry::getFromProtobuf, CompactionLogEntry::getProtobuf); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java index 29fd467ea06..8b9b76df895 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java @@ -58,7 +58,7 @@ protected SCMDBDefinition(Map> map) { Long.class, LongCodec.get(), DeletedBlocksTransaction.class, - Proto2Codec.get(DeletedBlocksTransaction.class)); + Proto2Codec.get(DeletedBlocksTransaction.getDefaultInstance())); public static final DBColumnFamilyDefinition VALID_CERTS = diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index cfef3693de6..cc811053eb2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java @@ -45,7 +45,7 @@ */ public final class OmBucketInfo extends WithObjectID implements Auditable { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(BucketInfo.class), + Proto2Codec.get(BucketInfo.getDefaultInstance()), OmBucketInfo::getFromProtobuf, OmBucketInfo::getProtobuf); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java index 53a1b7e291d..8ca0054b347 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java @@ -31,7 +31,7 @@ */ public final class OmDBAccessIdInfo { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(ExtendedUserAccessIdInfo.class), + Proto2Codec.get(ExtendedUserAccessIdInfo.getDefaultInstance()), OmDBAccessIdInfo::getFromProtobuf, OmDBAccessIdInfo::getProtobuf, DelegatedCodec.CopyType.SHALLOW); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java index 0c113aba85e..bb356eafdd9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java @@ -20,7 +20,7 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.Proto2Codec; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantState; import java.util.Objects; @@ -31,7 +31,7 @@ */ public final class OmDBTenantState implements Comparable { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(OzoneManagerProtocolProtos.TenantState.class), + Proto2Codec.get(TenantState.getDefaultInstance()), OmDBTenantState::getFromProtobuf, OmDBTenantState::getProtobuf, DelegatedCodec.CopyType.SHALLOW); @@ -112,7 +112,7 @@ public String getTenantId() { /** * Returns the bucket namespace name. a.k.a. volume name. - * + *

* Note: This returns an empty string ("") if the tenant is somehow not * associated with a volume. Should never return null. */ @@ -139,8 +139,8 @@ public String getBucketPolicyName() { /** * Convert OmDBTenantState to protobuf to be persisted to DB. */ - public OzoneManagerProtocolProtos.TenantState getProtobuf() { - return OzoneManagerProtocolProtos.TenantState.newBuilder() + public TenantState getProtobuf() { + return TenantState.newBuilder() .setTenantId(tenantId) .setBucketNamespaceName(bucketNamespaceName) .setUserRoleName(userRoleName) @@ -153,8 +153,7 @@ public OzoneManagerProtocolProtos.TenantState getProtobuf() { /** * Convert protobuf to OmDBTenantState. */ - public static OmDBTenantState getFromProtobuf( - OzoneManagerProtocolProtos.TenantState proto) { + public static OmDBTenantState getFromProtobuf(TenantState proto) { return new Builder() .setTenantId(proto.getTenantId()) .setBucketNamespaceName(proto.getBucketNamespaceName()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBUserPrincipalInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBUserPrincipalInfo.java index 273e5138ba2..75b01a04171 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBUserPrincipalInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBUserPrincipalInfo.java @@ -35,7 +35,7 @@ public final class OmDBUserPrincipalInfo { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(TenantUserPrincipalInfo.class), + Proto2Codec.get(TenantUserPrincipalInfo.getDefaultInstance()), OmDBUserPrincipalInfo::getFromProtobuf, OmDBUserPrincipalInfo::getProtobuf); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java index 66f33cf5b4e..3d1940bd7ce 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.utils.db.Proto2Codec; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DirectoryInfo; import java.util.BitSet; import java.util.HashMap; @@ -40,7 +40,7 @@ public class OmDirectoryInfo extends WithParentObjectId implements CopyObject { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(OzoneManagerProtocolProtos.DirectoryInfo.class), + Proto2Codec.get(DirectoryInfo.getDefaultInstance()), OmDirectoryInfo::getFromProtobuf, OmDirectoryInfo::getProtobuf); @@ -191,9 +191,9 @@ public List getAcls() { /** * Creates DirectoryInfo protobuf from OmDirectoryInfo. */ - public OzoneManagerProtocolProtos.DirectoryInfo getProtobuf() { - OzoneManagerProtocolProtos.DirectoryInfo.Builder pib = - OzoneManagerProtocolProtos.DirectoryInfo.newBuilder().setName(name) + public DirectoryInfo getProtobuf() { + final DirectoryInfo.Builder pib = + DirectoryInfo.newBuilder().setName(name) .setCreationTime(creationTime) .setModificationTime(modificationTime) .addAllMetadata(KeyValueUtil.toProtobuf(metadata)) @@ -211,8 +211,7 @@ public OzoneManagerProtocolProtos.DirectoryInfo getProtobuf() { * @param dirInfo * @return instance of OmDirectoryInfo */ - public static OmDirectoryInfo getFromProtobuf( - OzoneManagerProtocolProtos.DirectoryInfo dirInfo) { + public static OmDirectoryInfo getFromProtobuf(DirectoryInfo dirInfo) { OmDirectoryInfo.Builder opib = OmDirectoryInfo.newBuilder() .setName(dirInfo.getName()) .setCreationTime(dirInfo.getCreationTime()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index b3be2306ecd..c3a1a4a3d77 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -62,7 +62,7 @@ public final class OmKeyInfo extends WithParentObjectId private static Codec newCodec(boolean ignorePipeline) { return new DelegatedCodec<>( - Proto2Codec.get(KeyInfo.class), + Proto2Codec.get(KeyInfo.getDefaultInstance()), OmKeyInfo::getFromProtobuf, k -> k.getProtobuf(ignorePipeline, ClientVersion.CURRENT_VERSION)); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java index 48ea18045eb..4f57e075bd7 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java @@ -39,7 +39,7 @@ */ public final class OmMultipartKeyInfo extends WithObjectID { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(MultipartKeyInfo.class), + Proto2Codec.get(MultipartKeyInfo.getDefaultInstance()), OmMultipartKeyInfo::getFromProto, OmMultipartKeyInfo::getProto); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java index 37673c4f7ab..c5c8f5ca8e2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java @@ -44,7 +44,7 @@ public final class OmVolumeArgs extends WithObjectID implements CopyObject, Auditable { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(VolumeInfo.class), + Proto2Codec.get(VolumeInfo.getDefaultInstance()), OmVolumeArgs::getFromProtobuf, OmVolumeArgs::getProtobuf); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java index 2ee5420a4cd..24c172ef8fd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java @@ -45,7 +45,7 @@ public class RepeatedOmKeyInfo implements CopyObject { private static Codec newCodec(boolean ignorePipeline) { return new DelegatedCodec<>( - Proto2Codec.get(RepeatedKeyInfo.class), + Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()), RepeatedOmKeyInfo::getFromProto, k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION)); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java index 959f83a14f4..e97adc0a50f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java @@ -20,7 +20,7 @@ import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.Proto2Codec; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret; import java.util.Objects; @@ -29,7 +29,7 @@ */ public class S3SecretValue { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(OzoneManagerProtocolProtos.S3Secret.class), + Proto2Codec.get(S3Secret.getDefaultInstance()), S3SecretValue::fromProtobuf, S3SecretValue::getProtobuf); @@ -91,13 +91,12 @@ public void setTransactionLogIndex(long transactionLogIndex) { this.transactionLogIndex = transactionLogIndex; } - public static S3SecretValue fromProtobuf( - OzoneManagerProtocolProtos.S3Secret s3Secret) { + public static S3SecretValue fromProtobuf(S3Secret s3Secret) { return new S3SecretValue(s3Secret.getKerberosID(), s3Secret.getAwsSecret()); } - public OzoneManagerProtocolProtos.S3Secret getProtobuf() { - return OzoneManagerProtocolProtos.S3Secret.newBuilder() + public S3Secret getProtobuf() { + return S3Secret.newBuilder() .setAwsSecret(this.awsSecret) .setKerberosID(this.kerberosID) .build(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index a9065c03fe3..8ee9c6ee1f5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -55,7 +55,8 @@ */ public final class SnapshotInfo implements Auditable, CopyObject { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(OzoneManagerProtocolProtos.SnapshotInfo.class), + Proto2Codec.get( + OzoneManagerProtocolProtos.SnapshotInfo.getDefaultInstance()), SnapshotInfo::getFromProtobuf, SnapshotInfo::getProtobuf); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java index c04c09b219a..aec0c6d12cc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java @@ -44,7 +44,7 @@ public class SnapshotDiffReportOzone extends org.apache.hadoop.hdfs.protocol.SnapshotDiffReport { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(DiffReportEntryProto.class), + Proto2Codec.get(DiffReportEntryProto.getDefaultInstance()), SnapshotDiffReportOzone::fromProtobufDiffReportEntry, SnapshotDiffReportOzone::toProtobufDiffReportEntry, DelegatedCodec.CopyType.SHALLOW); diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java index cc8c36ed8b5..4ec5b952703 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java @@ -40,7 +40,7 @@ // TODO: support Auditable interface public final class OmPrefixInfo extends WithObjectID { private static final Codec CODEC = new DelegatedCodec<>( - Proto2Codec.get(PersistedPrefixInfo.class), + Proto2Codec.get(PersistedPrefixInfo.getDefaultInstance()), OmPrefixInfo::getFromProtobuf, OmPrefixInfo::getProtobuf); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 42179294e56..7a710115218 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -634,7 +634,7 @@ public static DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { .addCodec(RepeatedOmKeyInfo.class, RepeatedOmKeyInfo.getCodec(true)) .addCodec(OmBucketInfo.class, OmBucketInfo.getCodec()) .addCodec(OmVolumeArgs.class, OmVolumeArgs.getCodec()) - .addProto2Codec(PersistedUserVolumeInfo.class) + .addProto2Codec(PersistedUserVolumeInfo.getDefaultInstance()) .addCodec(OmMultipartKeyInfo.class, OmMultipartKeyInfo.getCodec()) .addCodec(S3SecretValue.class, S3SecretValue.getCodec()) .addCodec(OmPrefixInfo.class, OmPrefixInfo.getCodec()) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 056e1b015c8..de567447ae3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -62,13 +62,12 @@ public class OMDBDefinition extends DBDefinition.WithMap { RepeatedOmKeyInfo.getCodec(true)); public static final DBColumnFamilyDefinition - USER_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.USER_TABLE, - String.class, - StringCodec.get(), - PersistedUserVolumeInfo.class, - Proto2Codec.get(PersistedUserVolumeInfo.class)); + USER_TABLE = new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.USER_TABLE, + String.class, + StringCodec.get(), + PersistedUserVolumeInfo.class, + Proto2Codec.get(PersistedUserVolumeInfo.getDefaultInstance())); public static final DBColumnFamilyDefinition VOLUME_TABLE = diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistoryList.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistoryList.java index 0640ef46fe5..5895d3e133c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistoryList.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistoryList.java @@ -36,7 +36,7 @@ public class ContainerReplicaHistoryList { private static final Codec CODEC = new DelegatedCodec<>(Proto2Codec.get( - ContainerReplicaHistoryListProto.class), + ContainerReplicaHistoryListProto.getDefaultInstance()), ContainerReplicaHistoryList::fromProto, ContainerReplicaHistoryList::toProto); From 0ad306faadd1a58bec25b60885a74ac4833c32a0 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 29 Nov 2023 10:09:45 -0800 Subject: [PATCH 27/51] Migrated TestOmSnapshot to Junit5 (#5696) --- .../ozone/om/TestOmSnapshotFileSystem.java | 1 + .../om/{ => snapshot}/TestOmSnapshot.java | 492 ++++++++---------- .../TestOmSnapshotFsoWithNativeLib.java | 33 ++ .../TestOmSnapshotFsoWithoutNativeLib.java | 34 ++ .../om/snapshot/TestOmSnapshotLegacy.java | 34 ++ .../snapshot/TestOmSnapshotObjectStore.java | 34 ++ 6 files changed, 365 insertions(+), 263 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{ => snapshot}/TestOmSnapshot.java (87%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index bd384256792..7c7205b20b7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.snapshot.TestOmSnapshot; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java similarity index 87% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 83f0ff3144a..ff444a88ceb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -1,21 +1,23 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

+/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package org.apache.hadoop.ozone.om; +package org.apache.hadoop.ozone.om.snapshot; + import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; @@ -56,8 +58,17 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.KeyManagerImpl; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -76,23 +87,12 @@ import org.apache.log4j.Logger; import org.apache.ozone.rocksdiff.CompactionNode; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.SlowTest; -import org.apache.ozone.test.UnhealthyTest; import org.apache.ozone.test.tag.Slow; import org.apache.ozone.test.tag.Unhealthy; import org.jetbrains.annotations.NotNull; -import org.junit.Assert; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.jupiter.api.Assertions; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.rocksdb.LiveFileMetaData; import java.io.File; @@ -100,7 +100,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.nio.charset.StandardCharsets; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.Set; @@ -119,14 +118,13 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONTAINS_SNAPSHOT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; -import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; -import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_CANCELLED_JOB; @@ -138,87 +136,63 @@ import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; import static org.awaitility.Awaitility.with; import static org.awaitility.Awaitility.await; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertThrows; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertThrows; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** - * Test OmSnapshot bucket interface. + * Abstract class to test OmSnapshot. */ -@RunWith(Parameterized.class) -public class TestOmSnapshot { - +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class TestOmSnapshot { static { Logger.getLogger(ManagedRocksObjectUtils.class).setLevel(Level.DEBUG); } - private static MiniOzoneCluster cluster = null; - private static OzoneClient client; - private static String volumeName; - private static String bucketName; - private static OzoneManagerProtocol writeClient; - private static BucketLayout bucketLayout = BucketLayout.LEGACY; - private static boolean enabledFileSystemPaths; - private static boolean forceFullSnapshotDiff; - private static boolean disableNativeDiff; - private static ObjectStore store; - private static OzoneManager ozoneManager; - private static OzoneBucket ozoneBucket; - private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(500); - private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(10); - - private static AtomicInteger counter; private static final String SNAPSHOT_DAY_PREFIX = "snap-day-"; private static final String SNAPSHOT_WEEK_PREFIX = "snap-week-"; private static final String SNAPSHOT_MONTH_PREFIX = "snap-month-"; private static final String KEY_PREFIX = "key-"; private static final String SNAPSHOT_KEY_PATTERN_STRING = "(.+)/(.+)/(.+)"; - private static Pattern snapshotKeyPattern = + private static final Pattern SNAPSHOT_KEY_PATTERN = Pattern.compile(SNAPSHOT_KEY_PATTERN_STRING); + private static final Duration POLL_INTERVAL_DURATION = Duration.ofMillis(500); + private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(10); - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[]{OBJECT_STORE, false, false, false}, - new Object[]{FILE_SYSTEM_OPTIMIZED, false, false, false}, - new Object[]{FILE_SYSTEM_OPTIMIZED, false, false, true}, - new Object[]{BucketLayout.LEGACY, true, true, false}); - } + private MiniOzoneCluster cluster; + private OzoneClient client; + private String volumeName; + private String bucketName; + private OzoneManagerProtocol writeClient; + private ObjectStore store; + private OzoneManager ozoneManager; + private OzoneBucket ozoneBucket; + + private final BucketLayout bucketLayout; + private final boolean enabledFileSystemPaths; + private final boolean forceFullSnapshotDiff; + private final boolean disableNativeDiff; + private final AtomicInteger counter; public TestOmSnapshot(BucketLayout newBucketLayout, - boolean newEnableFileSystemPaths, boolean forceFullSnapDiff, - boolean disableNativeDiff) + boolean newEnableFileSystemPaths, + boolean forceFullSnapDiff, + boolean disableNativeDiff) throws Exception { - // Checking whether 'newBucketLayout' and - // 'newEnableFileSystemPaths' flags represents next parameter - // index values. This is to ensure that initialize init() function - // will be invoked only at the beginning of every new set of - // Parameterized.Parameters. - if (TestOmSnapshot.enabledFileSystemPaths != newEnableFileSystemPaths || - TestOmSnapshot.bucketLayout != newBucketLayout || - TestOmSnapshot.forceFullSnapshotDiff != forceFullSnapDiff || - TestOmSnapshot.disableNativeDiff - != disableNativeDiff) { - setConfig(newBucketLayout, newEnableFileSystemPaths, - forceFullSnapDiff, disableNativeDiff); - tearDown(); - init(); - } - } - - private static void setConfig(BucketLayout newBucketLayout, - boolean newEnableFileSystemPaths, boolean forceFullSnapDiff, - boolean forceNonNativeSnapDiff) { - TestOmSnapshot.enabledFileSystemPaths = newEnableFileSystemPaths; - TestOmSnapshot.bucketLayout = newBucketLayout; - TestOmSnapshot.forceFullSnapshotDiff = forceFullSnapDiff; - TestOmSnapshot.disableNativeDiff = forceNonNativeSnapDiff; + this.enabledFileSystemPaths = newEnableFileSystemPaths; + this.bucketLayout = newBucketLayout; + this.forceFullSnapshotDiff = forceFullSnapDiff; + this.disableNativeDiff = disableNativeDiff; + this.counter = new AtomicInteger(); + init(); } /** @@ -228,15 +202,11 @@ private void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - enabledFileSystemPaths); - conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, - bucketLayout.name()); - conf.setBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, - forceFullSnapshotDiff); - conf.setBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS, + conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths); + conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); + conf.setBoolean(OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff); + conf.setBoolean(OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS, disableNativeDiff); - String omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); conf.setBoolean(OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff); @@ -250,14 +220,13 @@ private void init() throws Exception { .setNumOfOzoneManagers(3) .setOmLayoutVersion(OMLayoutFeature. BUCKET_LAYOUT_SUPPORT.layoutVersion()) - .setOmId(omId) + .setOmId(UUID.randomUUID().toString()) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - ozoneBucket = TestDataUtil - .createVolumeAndBucket(client, bucketLayout); + ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout); volumeName = ozoneBucket.getVolumeName(); bucketName = ozoneBucket.getName(); ozoneManager = cluster.getOzoneManager(); @@ -265,26 +234,23 @@ private void init() throws Exception { store = client.getObjectStore(); writeClient = store.getClientProxy().getOzoneManagerClient(); + // stop the deletion services so that keys can still be read stopKeyManager(); -// preFinalizationChecks(); + preFinalizationChecks(); finalizeOMUpgrade(); - counter = new AtomicInteger(); } - private static void stopKeyManager() throws IOException { + private void stopKeyManager() throws IOException { KeyManagerImpl keyManager = (KeyManagerImpl) HddsWhiteboxTestUtils .getInternalState(ozoneManager, "keyManager"); - // stop the deletion services so that keys can still be read keyManager.stop(); } - private static RDBStore getRdbStore() { + private RDBStore getRdbStore() { return (RDBStore) ozoneManager.getMetadataManager().getStore(); } - private static void preFinalizationChecks() throws Exception { - // None of the snapshot APIs is usable before the upgrade finalization step - + private void preFinalizationChecks() { OMException omException = assertThrows(OMException.class, () -> store.createSnapshot(volumeName, bucketName, UUID.randomUUID().toString())); @@ -306,27 +272,26 @@ private static void preFinalizationChecks() throws Exception { } private static void assertFinalizationException(OMException omException) { - Assert.assertEquals(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, + assertEquals(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, omException.getResult()); - Assert.assertTrue(omException.getMessage().contains( - "cannot be invoked before finalization.")); + assertThat(omException.getMessage(), + containsString("cannot be invoked before finalization.")); } + /** * Trigger OM upgrade finalization from the client and block until completion * (status FINALIZATION_DONE). */ - private static void finalizeOMUpgrade() throws IOException { - + private void finalizeOMUpgrade() throws IOException { // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call - final OzoneManagerProtocol omclient = - client.getObjectStore() + final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); UpgradeFinalizer.StatusAndMessages finalizationResponse = - omclient.finalizeUpgrade(upgradeClientID); + omClient.finalizeUpgrade(upgradeClientID); // The status should transition as soon as the client call above returns - Assert.assertTrue(isStarting(finalizationResponse.status())); + assertTrue(isStarting(finalizationResponse.status())); // Wait for the finalization to be marked as done. // 10s timeout should be plenty. try { @@ -335,18 +300,18 @@ private static void finalizeOMUpgrade() throws IOException { .await() .until(() -> { final UpgradeFinalizer.StatusAndMessages progress = - omclient.queryUpgradeFinalizationProgress( + omClient.queryUpgradeFinalizationProgress( upgradeClientID, false, false); return isDone(progress.status()); }); } catch (Exception e) { - Assert.fail("Unexpected exception while waiting for " + fail("Unexpected exception while waiting for " + "the OM upgrade to finalize: " + e.getMessage()); } } - @AfterClass - public static void tearDown() throws Exception { + @AfterAll + void tearDown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); @@ -402,40 +367,40 @@ public void testListKey() throws Exception { int volABucketAKeyCount = keyCount(volAbucketA, snapshotKeyPrefix + "key-"); - Assert.assertEquals(20, volABucketAKeyCount); + assertEquals(20, volABucketAKeyCount); snapshotKeyPrefix = createSnapshot(volumeA, bucketB); deleteKeys(volAbucketB); int volABucketBKeyCount = keyCount(volAbucketB, snapshotKeyPrefix + "key-"); - Assert.assertEquals(20, volABucketBKeyCount); + assertEquals(20, volABucketBKeyCount); snapshotKeyPrefix = createSnapshot(volumeB, bucketA); deleteKeys(volBbucketA); int volBBucketAKeyCount = keyCount(volBbucketA, snapshotKeyPrefix + "key-"); - Assert.assertEquals(20, volBBucketAKeyCount); + assertEquals(20, volBBucketAKeyCount); snapshotKeyPrefix = createSnapshot(volumeB, bucketB); deleteKeys(volBbucketB); int volBBucketBKeyCount = keyCount(volBbucketB, snapshotKeyPrefix + "key-"); - Assert.assertEquals(20, volBBucketBKeyCount); + assertEquals(20, volBBucketBKeyCount); snapshotKeyPrefix = createSnapshot(volumeA, bucketA); deleteKeys(volAbucketA); int volABucketAKeyACount = keyCount(volAbucketA, snapshotKeyPrefix + "key-a-"); - Assert.assertEquals(10, volABucketAKeyACount); + assertEquals(10, volABucketAKeyACount); int volABucketAKeyBCount = keyCount(volAbucketA, snapshotKeyPrefix + "key-b-"); - Assert.assertEquals(10, volABucketAKeyBCount); + assertEquals(10, volABucketAKeyBCount); } @Test @@ -480,7 +445,6 @@ public void checkKey() throws Exception { ozoneOutputStream.write(input); ozoneOutputStream.close(); - String snapshotKeyPrefix = createSnapshot(volumeName, bucketName); GenericTestUtils.waitFor(() -> { @@ -489,9 +453,7 @@ public void checkKey() throws Exception { if (keyCount == 0) { return true; } - ozoneBucket.deleteKey(key1); - return false; } catch (Exception e) { return false; @@ -506,13 +468,14 @@ public void checkKey() throws Exception { fail("got exception on cleanup: " + e.getMessage()); } } + OmKeyArgs keyArgs = genKeyArgs(snapshotKeyPrefix + key1); - OmKeyInfo omKeyInfo = writeClient.lookupKey(keyArgs); - assertEquals(omKeyInfo.getKeyName(), snapshotKeyPrefix + key1); + KeyInfoWithVolumeContext omKeyInfo = writeClient.getKeyInfo(keyArgs, false); + assertEquals(omKeyInfo.getKeyInfo().getKeyName(), snapshotKeyPrefix + key1); - OmKeyInfo fileInfo = writeClient.lookupFile(keyArgs); - assertEquals(fileInfo.getKeyName(), snapshotKeyPrefix + key1); + KeyInfoWithVolumeContext fileInfo = writeClient.getKeyInfo(keyArgs, false); + assertEquals(fileInfo.getKeyInfo().getKeyName(), snapshotKeyPrefix + key1); OzoneFileStatus ozoneFileStatus = writeClient.getFileStatus(keyArgs); assertEquals(ozoneFileStatus.getKeyInfo().getKeyName(), @@ -534,7 +497,7 @@ public void testListDeleteKey() throws Exception { deleteKeys(volBucket); int volBucketKeyCount = keyCount(volBucket, snapshotKeyPrefix + "key-"); - Assert.assertEquals(1, volBucketKeyCount); + assertEquals(1, volBucketKeyCount); snapshotKeyPrefix = createSnapshot(volume, bucket); Iterator volBucketIter2 = @@ -562,11 +525,11 @@ public void testListAddNewKey() throws Exception { String snapshotKeyPrefix2 = createSnapshot(volume, bucket); int volBucketKeyCount = keyCount(bucket1, snapshotKeyPrefix1 + "key-"); - Assert.assertEquals(1, volBucketKeyCount); + assertEquals(1, volBucketKeyCount); int volBucketKeyCount2 = keyCount(bucket1, snapshotKeyPrefix2 + "key-"); - Assert.assertEquals(2, volBucketKeyCount2); + assertEquals(2, volBucketKeyCount2); deleteKeys(bucket1); } @@ -641,16 +604,16 @@ private Set getDeletedKeysFromRocksDb( return deletedKeys; } - private OmKeyInfo getOmKeyInfo(String volume, String bucket, - String key) throws IOException { + private void getOmKeyInfo(String volume, String bucket, + String key) throws IOException { ResolvedBucket resolvedBucket = new ResolvedBucket(volume, bucket, volume, bucket, "", bucketLayout); - return cluster.getOzoneManager().getKeyManager() - .getKeyInfo(new OmKeyArgs.Builder() + cluster.getOzoneManager().getKeyManager() + .getKeyInfo(new OmKeyArgs.Builder() .setVolumeName(volume) .setBucketName(bucket) .setKeyName(key).build(), - resolvedBucket, null); + resolvedBucket, null); } /** @@ -682,10 +645,10 @@ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception { createSnapshot(testVolumeName, testBucketName, snap3); SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); - Assert.assertEquals(diff.getDiffList().size(), 0); + assertEquals(diff.getDiffList().size(), 0); diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1))); } @@ -697,8 +660,8 @@ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception { * 3) Key k1 is deleted. * 4) Snapshot snap2 is created. * 5) Snapshot snap3 is created. - * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key. - * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key. + * 6) Snap diff b/w snap3 & snap1 taken to assert difference of 1 key. + * 7) Snap diff b/w snap3 & snap2 taken to assert difference of 0 key. */ @Test public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception { @@ -720,12 +683,12 @@ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception { createSnapshot(testVolumeName, testBucketName, snap3); SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap3); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1))); diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); - Assert.assertEquals(diff.getDiffList().size(), 0); + assertEquals(diff.getDiffList().size(), 0); } /** @@ -735,16 +698,16 @@ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception { * 3) Key k1 is deleted. * 4) Key k1 is recreated. * 5) Snapshot snap2 is created. - * 6) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 6) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 2 keys. - * 7) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * 7) Snap diff b/w snapshot of Active FS & snap2 taken to assert difference * of 0 key. * 8) Checking rocks db to ensure the object created shouldn't be reclaimed * as it is used by snapshot. * 9) Key k1 is deleted. - * 10) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 10) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 1 key. - * 11) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * 11) Snap diff b/w snapshot of Active FS & snap2 taken to assert difference * of 1 key. */ @Test @@ -769,25 +732,25 @@ public void testSnapDiffReclaimWithKeyRecreation() throws Exception { createSnapshot(testVolumeName, testBucketName, snap3); SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap3); - Assert.assertEquals(diff.getDiffList().size(), 2); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 2); + assertEquals(diff.getDiffList(), Arrays.asList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1), SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.CREATE, key1))); diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); - Assert.assertEquals(diff.getDiffList().size(), 0); + assertEquals(diff.getDiffList().size(), 0); bucket.deleteKey(key1); String snap4 = "snap4"; createSnapshot(testVolumeName, testBucketName, snap4); diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap4); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1))); diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap4); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1))); } @@ -799,7 +762,7 @@ public void testSnapDiffReclaimWithKeyRecreation() throws Exception { * 3) Key k1 is renamed to renamed-k1. * 4) Key renamed-k1 is deleted. * 5) Snapshot snap2 created. - * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key. + * 4) Snap diff b/w snap2 & snap1 taken to assert difference of 1 key. */ @Test public void testSnapDiffReclaimWithKeyRename() throws Exception { @@ -829,8 +792,8 @@ public void testSnapDiffReclaimWithKeyRename() throws Exception { createSnapshot(testVolumeName, testBucketName, snap2); SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1) )); @@ -844,7 +807,7 @@ public void testSnapDiffReclaimWithKeyRename() throws Exception { * 4) Key renamed-k1 is renamed to renamed-renamed-k1. * 5) Key renamed-renamed-k1 is deleted. * 6) Snapshot snap2 is created. - * 7) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key. + * 7) Snap diff b/w Active FS & snap1 taken to assert difference of 1 key. */ @Test public void testSnapDiffWith2RenamesAndDelete() throws Exception { @@ -887,8 +850,8 @@ public void testSnapDiffWith2RenamesAndDelete() throws Exception { snap3); SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap3); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1) )); @@ -901,7 +864,7 @@ public void testSnapDiffWith2RenamesAndDelete() throws Exception { * 3) Key k1 is renamed to renamed-k1. * 4) Key k1 is recreated. * 5) Key k1 is deleted. - * 6) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 6) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 1 key. */ @Test @@ -930,8 +893,8 @@ public void testSnapDiffWithKeyRenamesRecreationAndDelete() createSnapshot(testVolumeName, testBucketName, activeSnap); SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, activeSnap); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, key1, renamedKey) @@ -944,9 +907,9 @@ public void testSnapDiffWithKeyRenamesRecreationAndDelete() * 2) Key k1 is created. * 3) Key k1 is deleted. * 4) Snapshot s2 is created before key k1 is reclaimed. - * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 5) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 0 keys. - * 6) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * 6) Snap diff b/w snapshot of Active FS & snap2 taken to assert difference * of 0 keys. */ @Test @@ -969,9 +932,9 @@ public void testSnapDiffReclaimWithDeferredKeyDeletion() throws Exception { createSnapshot(testVolumeName, testBucketName, activeSnap); SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, activeSnap); - Assert.assertEquals(diff.getDiffList().size(), 0); + assertEquals(diff.getDiffList().size(), 0); diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, activeSnap); - Assert.assertEquals(diff.getDiffList().size(), 0); + assertEquals(diff.getDiffList().size(), 0); } /** @@ -980,7 +943,7 @@ public void testSnapDiffReclaimWithDeferredKeyDeletion() throws Exception { * 2) Snapshot snap1 created. * 3) Key k1 is renamed to key k1_renamed * 4) Key k1_renamed is renamed to key k1 - * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 5) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 1 key with 1 Modified entry. */ @Test @@ -1009,7 +972,7 @@ public void testSnapDiffWithNoEffectiveRename() throws Exception { snap1, snap2); getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), key1); - Assert.assertEquals(diff.getDiffList(), Arrays.asList( + assertEquals(diff.getDiffList(), Collections.singletonList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, key1, key1))); } @@ -1020,7 +983,7 @@ public void testSnapDiffWithNoEffectiveRename() throws Exception { * 2) Snapshot snap1 created. * 3) Dir dir1/dir2 is created. * 4) Key k1 is renamed to key dir1/dir2/k1_renamed - * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * 5) Snap diff b/w snapshot of Active FS & snap1 taken to assert difference * of 3 key * with 1 rename entry & 2 dirs create entry. */ @@ -1068,7 +1031,7 @@ public void testSnapDiffWithDirectory() throws Exception { SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.CREATE, "dir1")); } - Assert.assertEquals(diff.getDiffList(), diffEntries); + assertEquals(diff.getDiffList(), diffEntries); } /** @@ -1084,7 +1047,7 @@ public void testSnapDiffWithDirectory() throws Exception { */ @Test public void testSnapDiffWithDirectoryDelete() throws Exception { - Assume.assumeTrue(bucketLayout.isFileSystemOptimized()); + assumeTrue(bucketLayout.isFileSystemOptimized()); String testVolumeName = "vol" + counter.incrementAndGet(); String testBucketName = "bucket1"; store.createVolume(testVolumeName); @@ -1107,7 +1070,7 @@ public void testSnapDiffWithDirectoryDelete() throws Exception { List diffEntries = Lists.newArrayList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1)); - Assert.assertEquals(diff.getDiffList(), diffEntries); + assertEquals(diff.getDiffList(), diffEntries); } private OzoneObj buildKeyObj(OzoneBucket bucket, String key) { @@ -1144,7 +1107,7 @@ public void testSnapdiffWithObjectMetaModification() throws Exception { SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.MODIFY, key1)); - Assert.assertEquals(diffEntries, diff.getDiffList()); + assertEquals(diffEntries, diff.getDiffList()); } @Test @@ -1152,7 +1115,7 @@ public void testSnapdiffWithFilesystemCreate() throws IOException, URISyntaxException, InterruptedException, TimeoutException { - Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); String testVolumeName = "vol" + counter.incrementAndGet(); String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); @@ -1174,12 +1137,12 @@ public void testSnapdiffWithFilesystemCreate() Path p = new Path("/"); while (true) { FileStatus[] fileStatuses = fs.listStatus(p); - Assertions.assertEquals(fileStatuses[0].isDirectory(), + assertEquals(fileStatuses[0].isDirectory(), bucketLayout.isFileSystemOptimized() && idx < diff.getDiffList().size() - 1 || !bucketLayout.isFileSystemOptimized() && idx > 0); p = fileStatuses[0].getPath(); - Assertions.assertEquals(diff.getDiffList().get(idx), + assertEquals(diff.getDiffList().get(idx), SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.CREATE, p.toUri().getPath() .substring(1))); @@ -1192,10 +1155,10 @@ public void testSnapdiffWithFilesystemCreate() } @Test - public void testSnapdiffWithFilesystemDirectoryRenameOperation() + public void testSnapDiffWithFilesystemDirectoryRenameOperation() throws IOException, URISyntaxException, InterruptedException, TimeoutException { - Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); String testVolumeName = "vol" + counter.incrementAndGet(); String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); @@ -1214,14 +1177,14 @@ public void testSnapdiffWithFilesystemDirectoryRenameOperation() SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); if (bucketLayout.isFileSystemOptimized()) { - Assertions.assertEquals(Arrays.asList( + assertEquals(Arrays.asList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir1/dir3"), SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.MODIFY, "dir1")), diff.getDiffList()); } else { - Assertions.assertEquals(Arrays.asList( + assertEquals(Arrays.asList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1", "dir1/dir3/key1"), @@ -1235,10 +1198,10 @@ public void testSnapdiffWithFilesystemDirectoryRenameOperation() } @Test - public void testSnapdiffWithFilesystemDirectoryMoveOperation() + public void testSnapDiffWithFilesystemDirectoryMoveOperation() throws IOException, URISyntaxException, InterruptedException, TimeoutException { - Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); String testVolumeName = "vol" + counter.incrementAndGet(); String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); @@ -1258,7 +1221,7 @@ public void testSnapdiffWithFilesystemDirectoryMoveOperation() SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); if (bucketLayout.isFileSystemOptimized()) { - Assertions.assertEquals(Arrays.asList( + assertEquals(Arrays.asList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir3/dir2"), SnapshotDiffReportOzone.getDiffReportEntry( @@ -1267,7 +1230,7 @@ public void testSnapdiffWithFilesystemDirectoryMoveOperation() SnapshotDiffReport.DiffType.MODIFY, "dir3")), diff.getDiffList()); } else { - Assertions.assertEquals(Arrays.asList( + assertEquals(Arrays.asList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1", "dir3/dir2/key1"), @@ -1296,9 +1259,9 @@ public void testBucketDeleteIfSnapshotExists() throws Exception { createSnapshot(volume1, bucket1); deleteKeys(bucketWithSnapshot); deleteKeys(bucketWithoutSnapshot); - OMException omException = Assertions.assertThrows(OMException.class, + OMException omException = assertThrows(OMException.class, () -> volume.deleteBucket(bucket1)); - Assertions.assertEquals(CONTAINS_SNAPSHOT, omException.getResult()); + assertEquals(CONTAINS_SNAPSHOT, omException.getResult()); // TODO: Delete snapshot then delete bucket1 when deletion is implemented // no exception for bucket without snapshot volume.deleteBucket(bucket2); @@ -1323,14 +1286,14 @@ public void testGetSnapshotInfo() throws Exception { OzoneSnapshot snapshot1 = store.getSnapshotInfo(volume, bucket, snap1); - Assertions.assertEquals(snap1, snapshot1.getName()); - Assertions.assertEquals(volume, snapshot1.getVolumeName()); - Assertions.assertEquals(bucket, snapshot1.getBucketName()); + assertEquals(snap1, snapshot1.getName()); + assertEquals(volume, snapshot1.getVolumeName()); + assertEquals(bucket, snapshot1.getBucketName()); OzoneSnapshot snapshot2 = store.getSnapshotInfo(volume, bucket, snap2); - Assertions.assertEquals(snap2, snapshot2.getName()); - Assertions.assertEquals(volume, snapshot2.getVolumeName()); - Assertions.assertEquals(bucket, snapshot2.getBucketName()); + assertEquals(snap2, snapshot2.getName()); + assertEquals(volume, snapshot2.getVolumeName()); + assertEquals(bucket, snapshot2.getBucketName()); testGetSnapshotInfoFailure(null, bucket, "snapshotName", "volume can't be null or empty."); @@ -1348,14 +1311,14 @@ public void testGetSnapshotInfoFailure(String volName, String buckName, String snapName, String expectedMessage) { - Exception ioException = Assertions.assertThrows(Exception.class, + Exception ioException = assertThrows(Exception.class, () -> store.getSnapshotInfo(volName, buckName, snapName)); - Assertions.assertEquals(expectedMessage, ioException.getMessage()); + assertEquals(expectedMessage, ioException.getMessage()); } @Test public void testSnapDiffWithDirRename() throws Exception { - Assume.assumeTrue(bucketLayout.isFileSystemOptimized()); + assumeTrue(bucketLayout.isFileSystemOptimized()); String volume = "vol-" + counter.incrementAndGet(); String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); @@ -1370,9 +1333,9 @@ public void testSnapDiffWithDirRename() throws Exception { createSnapshot(volume, bucket, snap2); SnapshotDiffReportOzone diff = getSnapDiffReport(volume, bucket, snap1, snap2); - Assertions.assertEquals(Arrays.asList( - SnapshotDiffReportOzone.getDiffReportEntry( - SnapshotDiffReport.DiffType.RENAME, "dir1", "dir1_rename")), + assertEquals(Collections.singletonList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1", "dir1_rename")), diff.getDiffList()); } @@ -1401,7 +1364,7 @@ public void testSnapDiff() throws Exception { SnapshotDiffReportOzone diff1 = getSnapDiffReport(volume, bucket, snap1, snap2); - Assert.assertTrue(diff1.getDiffList().isEmpty()); + assertTrue(diff1.getDiffList().isEmpty()); // Create Key2 and delete Key1, take snapshot String key2 = "key-2-"; key2 = createFileKeyWithPrefix(bucket1, key2); @@ -1412,8 +1375,8 @@ public void testSnapDiff() throws Exception { // Diff should have 2 entries SnapshotDiffReportOzone diff2 = getSnapDiffReport(volume, bucket, snap2, snap3); - Assert.assertEquals(2, diff2.getDiffList().size()); - Assert.assertEquals( + assertEquals(2, diff2.getDiffList().size()); + assertEquals( Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.DELETE, key1), SnapshotDiffReportOzone.getDiffReportEntry( @@ -1428,8 +1391,8 @@ public void testSnapDiff() throws Exception { SnapshotDiffReportOzone diff3 = getSnapDiffReport(volume, bucket, snap3, snap4); - Assert.assertEquals(1, diff3.getDiffList().size()); - Assert.assertTrue(diff3.getDiffList().contains( + assertEquals(1, diff3.getDiffList().size()); + assertTrue(diff3.getDiffList().contains( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReportOzone.DiffType.RENAME, key2, key2Renamed))); @@ -1442,8 +1405,8 @@ public void testSnapDiff() throws Exception { createSnapshot(volume, bucket, snap5); SnapshotDiffReportOzone diff4 = getSnapDiffReport(volume, bucket, snap4, snap5); - Assert.assertEquals(1, diff4.getDiffList().size()); - Assert.assertTrue(diff4.getDiffList().contains( + assertEquals(1, diff4.getDiffList().size()); + assertTrue(diff4.getDiffList().contains( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReportOzone.DiffType.CREATE, dir1))); @@ -1565,25 +1528,25 @@ public void testSnapDiffCancelFailureResponses() throws Exception { volumeName, bucketName, fromSnapName, toSnapName, null, 0, false, disableNativeDiff); - Assert.assertEquals(IN_PROGRESS, response.getJobStatus()); + assertEquals(IN_PROGRESS, response.getJobStatus()); // Cancel success. cancelResponse = store.cancelSnapshotDiff(volumeName, bucketName, fromSnapName, toSnapName); - Assert.assertEquals(CANCEL_SUCCEEDED.getMessage(), + assertEquals(CANCEL_SUCCEEDED.getMessage(), cancelResponse.getMessage()); response = store.snapshotDiff( volumeName, bucketName, fromSnapName, toSnapName, null, 0, false, disableNativeDiff); - Assert.assertEquals(CANCELLED, response.getJobStatus()); + assertEquals(CANCELLED, response.getJobStatus()); // Job already cancelled. cancelResponse = store.cancelSnapshotDiff(volumeName, bucketName, fromSnapName, toSnapName); - Assert.assertEquals(CANCEL_ALREADY_CANCELLED_JOB.getMessage(), + assertEquals(CANCEL_ALREADY_CANCELLED_JOB.getMessage(), cancelResponse.getMessage()); } @@ -1692,8 +1655,8 @@ public void testSnapDiffWithKeyOverwrite() throws Exception { SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); getOmKeyInfo(testVolumeName, testBucketName, key1); - Assert.assertEquals(diff.getDiffList().size(), 1); - Assert.assertEquals(diff.getDiffList(), Lists.newArrayList( + assertEquals(diff.getDiffList().size(), 1); + assertEquals(diff.getDiffList(), Lists.newArrayList( SnapshotDiffReportOzone.getDiffReportEntry( SnapshotDiffReport.DiffType.MODIFY, key1))); } @@ -1758,7 +1721,7 @@ public void testSnapDiffMultipleBuckets() throws Exception { createSnapshot(volume, bucketName1, snap2); SnapshotDiffReportOzone diff1 = getSnapDiffReport(volume, bucketName1, snap1, snap2); - Assert.assertEquals(1, diff1.getDiffList().size()); + assertEquals(1, diff1.getDiffList().size()); } @Test @@ -1770,10 +1733,10 @@ public void testListSnapshotDiffWithInvalidParameters() String volBucketErrorMessage = "Provided volume name " + volume + " or bucket name " + bucket + " doesn't exist"; - Exception volBucketEx = Assertions.assertThrows(OMException.class, + Exception volBucketEx = assertThrows(OMException.class, () -> store.listSnapshotDiffJobs(volume, bucket, "", true)); - Assertions.assertEquals(volBucketErrorMessage, + assertEquals(volBucketErrorMessage, volBucketEx.getMessage()); // Create the volume and the bucket. @@ -1781,11 +1744,11 @@ public void testListSnapshotDiffWithInvalidParameters() OzoneVolume ozVolume = store.getVolume(volume); ozVolume.createBucket(bucket); - Assertions.assertDoesNotThrow(() -> + assertDoesNotThrow(() -> store.listSnapshotDiffJobs(volume, bucket, "", true)); // There are no snapshots, response should be empty. - Assertions.assertTrue(store + assertTrue(store .listSnapshotDiffJobs(volume, bucket, "", true).isEmpty()); @@ -1807,20 +1770,19 @@ public void testListSnapshotDiffWithInvalidParameters() String invalidStatus = "invalid"; String statusErrorMessage = "Invalid job status: " + invalidStatus; - Exception statusEx = Assertions.assertThrows(OMException.class, + OMException statusEx = assertThrows(OMException.class, () -> store.listSnapshotDiffJobs(volume, bucket, invalidStatus, false)); - Assertions.assertEquals(statusErrorMessage, - statusEx.getMessage()); + assertEquals(statusErrorMessage, statusEx.getMessage()); } /** - * Tests snapdiff when there are multiple sst files in the from & to + * Tests snap diff when there are multiple sst files in from and to * snapshots pertaining to different buckets. This will test the * sst filtering code path. */ @Test - @Category(UnhealthyTest.class) @Unhealthy("HDDS-8005") + @Unhealthy("HDDS-8005") public void testSnapDiffWithMultipleSSTs() throws Exception { // Create a volume and 2 buckets @@ -1838,28 +1800,28 @@ public void testSnapDiffWithMultipleSSTs() createFileKeyWithPrefix(bucket1, keyPrefix); String snap1 = "snap" + counter.incrementAndGet(); createSnapshot(volumeName1, bucketName1, snap1); // 1.sst - Assert.assertEquals(1, getKeyTableSstFiles().size()); + assertEquals(1, getKeyTableSstFiles().size()); // add files to bucket2 and flush twice to create 2 sst files for (int i = 0; i < 5; i++) { createFileKeyWithPrefix(bucket2, keyPrefix); } flushKeyTable(); // 1.sst 2.sst - Assert.assertEquals(2, getKeyTableSstFiles().size()); + assertEquals(2, getKeyTableSstFiles().size()); for (int i = 0; i < 5; i++) { createFileKeyWithPrefix(bucket2, keyPrefix); } flushKeyTable(); // 1.sst 2.sst 3.sst - Assert.assertEquals(3, getKeyTableSstFiles().size()); + assertEquals(3, getKeyTableSstFiles().size()); // add a file to bucket1 and take second snapshot createFileKeyWithPrefix(bucket1, keyPrefix); String snap2 = "snap" + counter.incrementAndGet(); createSnapshot(volumeName1, bucketName1, snap2); // 1.sst 2.sst 3.sst 4.sst - Assert.assertEquals(4, getKeyTableSstFiles().size()); + assertEquals(4, getKeyTableSstFiles().size()); SnapshotDiffReportOzone diff1 = store.snapshotDiff(volumeName1, bucketName1, snap1, snap2, null, 0, forceFullSnapshotDiff, disableNativeDiff) .getSnapshotDiffReport(); - Assert.assertEquals(1, diff1.getDiffList().size()); + assertEquals(1, diff1.getDiffList().size()); } @Test @@ -1986,7 +1948,7 @@ public void testSnapshotQuotaHandling() throws Exception { } @NotNull - private static List getKeyTableSstFiles() + private List getKeyTableSstFiles() throws IOException { if (!bucketLayout.isFileSystemOptimized()) { return getRdbStore().getDb().getSstFileList().stream().filter( @@ -1998,7 +1960,7 @@ private static List getKeyTableSstFiles() OmMetadataManagerImpl.FILE_TABLE)).collect(Collectors.toList()); } - private static void flushKeyTable() throws IOException { + private void flushKeyTable() throws IOException { if (!bucketLayout.isFileSystemOptimized()) { getRdbStore().getDb().flush(OmMetadataManagerImpl.KEY_TABLE); } else { @@ -2059,8 +2021,8 @@ private String createFileKey(OzoneBucket bucket, String key) return key; } - private String createFileKey(FileSystem fs, - String path) + private void createFileKey(FileSystem fs, + String path) throws IOException, InterruptedException, TimeoutException { byte[] value = RandomStringUtils.randomAscii(10240).getBytes(UTF_8); Path pathVal = new Path(path); @@ -2075,21 +2037,20 @@ private String createFileKey(FileSystem fs, } return true; }, 1000, 30000); - return path; } @Test public void testSnapshotOpensWithDisabledAutoCompaction() throws Exception { String snapPrefix = createSnapshot(volumeName, bucketName); - RDBStore snapshotDBStore = (RDBStore) + try (RDBStore snapshotDBStore = (RDBStore) ((OmSnapshot) cluster.getOzoneManager().getOmSnapshotManager() .checkForSnapshot(volumeName, bucketName, snapPrefix, false).get()) - .getMetadataManager().getStore(); - - for (String table : snapshotDBStore.getTableNames().values()) { - Assertions.assertTrue(snapshotDBStore.getDb().getColumnFamily(table) - .getHandle().getDescriptor() - .getOptions().disableAutoCompactions()); + .getMetadataManager().getStore()) { + for (String table : snapshotDBStore.getTableNames().values()) { + assertTrue(snapshotDBStore.getDb().getColumnFamily(table) + .getHandle().getDescriptor() + .getOptions().disableAutoCompactions()); + } } } @@ -2223,7 +2184,7 @@ public void testCompactionDagDisableForSnapshotMetadata() throws Exception { } @Test - @Category(SlowTest.class) @Slow("HDDS-9299") + @Slow("HDDS-9299") public void testDayWeekMonthSnapshotCreationAndExpiration() throws Exception { String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5); String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5); @@ -2288,15 +2249,20 @@ private int[] checkSnapshotExpirationThenCreateLatest(String snapshotPrefix, throws Exception { int targetOldestIndex = 0; int targetLatestIndex = 0; - if (snapshotPrefix.equals(SNAPSHOT_DAY_PREFIX)) { + switch (snapshotPrefix) { + case SNAPSHOT_DAY_PREFIX: targetOldestIndex = oldestDayIndex; targetLatestIndex = latestDayIndex; - } else if (snapshotPrefix.equals(SNAPSHOT_WEEK_PREFIX)) { + break; + case SNAPSHOT_WEEK_PREFIX: targetOldestIndex = oldestWeekIndex; targetLatestIndex = latestWeekIndex; - } else if (snapshotPrefix.equals(SNAPSHOT_MONTH_PREFIX)) { + break; + case SNAPSHOT_MONTH_PREFIX: targetOldestIndex = oldestMonthIndex; targetLatestIndex = latestMonthIndex; + break; + default: } targetOldestIndex = deleteOldestSnapshot(targetOldestIndex, targetLatestIndex, snapshotPrefix, snapshotRetentionPeriod, @@ -2350,10 +2316,10 @@ private void checkDayWeekMonthSnapshotData(OzoneBucket ozoneBucketClient, String keyName = KEY_PREFIX + i; // Validate keys metadata in active Ozone namespace OzoneKeyDetails ozoneKeyDetails = ozoneBucketClient.getKey(keyName); - Assert.assertEquals(keyName, ozoneKeyDetails.getName()); - Assert.assertEquals(ozoneBucketClient.getName(), + assertEquals(keyName, ozoneKeyDetails.getName()); + assertEquals(ozoneBucketClient.getName(), ozoneKeyDetails.getBucketName()); - Assert.assertEquals(ozoneBucketClient.getVolumeName(), + assertEquals(ozoneBucketClient.getVolumeName(), ozoneKeyDetails.getVolumeName()); // Validate keys data in active Ozone namespace @@ -2361,7 +2327,7 @@ private void checkDayWeekMonthSnapshotData(OzoneBucket ozoneBucketClient, ozoneBucketClient.readKey(keyName)) { byte[] fileContent = new byte[keyName.length()]; ozoneInputStream.read(fileContent); - Assert.assertEquals(keyName, new String(fileContent, UTF_8)); + assertEquals(keyName, new String(fileContent, UTF_8)); } } // Validate history day snapshot data integrity @@ -2395,12 +2361,12 @@ private void validateSnapshotDataIntegrity(String snapshotPrefix, // Thus need to remove snapshot related prefix from key name by regex // before use key name to compare key content // e.g.,".snapshot/snap-day-1/key-0" -> "key-0" - Matcher snapshotKeyNameMatcher = snapshotKeyPattern.matcher(keyName); - if (snapshotKeyNameMatcher.matches()) { - String truncatedSnapshotKeyName = snapshotKeyNameMatcher.group(3); + Matcher snapKeyNameMatcher = SNAPSHOT_KEY_PATTERN.matcher(keyName); + if (snapKeyNameMatcher.matches()) { + String truncatedSnapshotKeyName = snapKeyNameMatcher.group(3); byte[] fileContent = new byte[truncatedSnapshotKeyName.length()]; ozoneInputStream.read(fileContent); - Assert.assertEquals(truncatedSnapshotKeyName, + assertEquals(truncatedSnapshotKeyName, new String(fileContent, UTF_8)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java new file mode 100644 index 00000000000..60e66923524 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; + +/** + * Test OmSnapshot for FSO bucket type when native lib is enabled. + */ +@Timeout(300) +public class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { + public TestOmSnapshotFsoWithNativeLib() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java new file mode 100644 index 00000000000..c1782b73d19 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; + +/** + * Test OmSnapshot for FSO bucket type when native lib is disabled. + */ +@Timeout(300) +public class TestOmSnapshotFsoWithoutNativeLib extends TestOmSnapshot { + + public TestOmSnapshotFsoWithoutNativeLib() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, true); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java new file mode 100644 index 00000000000..bf4a2fee0de --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY; + +/** + * Test OmSnapshot for Legacy bucket type. + */ +@Timeout(300) +public class TestOmSnapshotLegacy extends TestOmSnapshot { + + public TestOmSnapshotLegacy() throws Exception { + super(LEGACY, false, false, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java new file mode 100644 index 00000000000..13c8cb5fca3 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; + +/** + * Test OmSnapshot for Object Store bucket type. + */ +@Timeout(300) +public class TestOmSnapshotObjectStore extends TestOmSnapshot { + + public TestOmSnapshotObjectStore() throws Exception { + super(OBJECT_STORE, false, false, false); + } +} From d83497827d6beb0a2b8c57061bbe9c0cfe13ee44 Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Thu, 30 Nov 2023 00:41:03 +0530 Subject: [PATCH 28/51] HDDS-9801. Increase the timeout in TestPipelineClose#testPipelineCloseWithPipelineAction. (#5702) --- .../org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java index a5ca909b3e7..b823f15798f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java @@ -199,7 +199,7 @@ public void testPipelineCloseWithPipelineAction() throws Exception { } } return true; - }, 500, 5000); + }, 500, 10000); assertThrows(PipelineNotFoundException.class, () -> pipelineManager.getPipeline(pipelineID), From d58e0823951e027a99482a52704f46e88cedf82f Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Wed, 29 Nov 2023 21:22:04 +0100 Subject: [PATCH 29/51] HDDS-9675. Eliminate unnecessary stream in SCMBlockProtocolServer#sortDatanodes (#5691) --- .../hadoop/hdds/scm/net/NetworkTopology.java | 4 +- .../hdds/scm/net/NetworkTopologyImpl.java | 42 ++++++++----------- .../scm/server/SCMBlockProtocolServer.java | 7 +--- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java index d0bbd17b51c..5aa9e17825c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java @@ -244,6 +244,6 @@ Node getNode(int leafIndex, String scope, List excludedScopes, * or shuffled input nodes otherwise. The size of returned list is limited * by activeLen parameter. */ - List sortByDistanceCost(Node reader, - List nodes, int activeLen); + List sortByDistanceCost(Node reader, + List nodes, int activeLen); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java index c05b30c6c78..5e0697eaafd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReadWriteLock; @@ -61,7 +62,7 @@ public class NetworkTopologyImpl implements NetworkTopology { /** The algorithm to randomize nodes with equal distances. */ private final Consumer> shuffleOperation; /** Lock to coordinate cluster tree access. */ - private ReadWriteLock netlock = new ReentrantReadWriteLock(true); + private final ReadWriteLock netlock = new ReentrantReadWriteLock(true); public NetworkTopologyImpl(ConfigurationSource conf) { schemaManager = NodeSchemaManager.getInstance(); @@ -136,7 +137,7 @@ public void add(Node node) { @Override public void update(Node oldNode, Node newNode) { Preconditions.checkArgument(newNode != null, "newNode cannot be null"); - if (oldNode != null && oldNode instanceof InnerNode) { + if (oldNode instanceof InnerNode) { throw new IllegalArgumentException( "Not allowed to update an inner node: " + oldNode.getNetworkFullPath()); @@ -225,10 +226,7 @@ private boolean containsNode(Node node) { while (parent != null && parent != clusterTree) { parent = parent.getParent(); } - if (parent == clusterTree) { - return true; - } - return false; + return parent == clusterTree; } /** @@ -384,7 +382,7 @@ public Node chooseRandom(String scope) { scope = ROOT; } if (scope.startsWith(SCOPE_REVERSE_STR)) { - ArrayList excludedScopes = new ArrayList(); + ArrayList excludedScopes = new ArrayList<>(); excludedScopes.add(scope.substring(1)); return chooseRandom(ROOT, excludedScopes, null, null, ANCESTOR_GENERATION_DEFAULT); @@ -425,7 +423,7 @@ public Node chooseRandom(String scope, Collection excludedNodes) { scope = ROOT; } if (scope.startsWith(SCOPE_REVERSE_STR)) { - ArrayList excludedScopes = new ArrayList(); + ArrayList excludedScopes = new ArrayList<>(); excludedScopes.add(scope.substring(1)); return chooseRandom(ROOT, excludedScopes, excludedNodes, null, ANCESTOR_GENERATION_DEFAULT); @@ -460,7 +458,7 @@ public Node chooseRandom(String scope, Collection excludedNodes, scope = ROOT; } if (scope.startsWith(SCOPE_REVERSE_STR)) { - ArrayList excludedScopes = new ArrayList(); + ArrayList excludedScopes = new ArrayList<>(); excludedScopes.add(scope.substring(1)); return chooseRandom(ROOT, excludedScopes, excludedNodes, null, ancestorGen); @@ -771,11 +769,11 @@ public int getDistanceCost(Node node1, Node node2) { * by activeLen parameter. */ @Override - public List sortByDistanceCost(Node reader, - List nodes, int activeLen) { + public List sortByDistanceCost(Node reader, + List nodes, int activeLen) { // shuffle input list of nodes if reader is not defined if (reader == null) { - List shuffledNodes = + List shuffledNodes = new ArrayList<>(nodes.subList(0, activeLen)); shuffleOperation.accept(shuffledNodes); return shuffledNodes; @@ -786,25 +784,19 @@ public List sortByDistanceCost(Node reader, costs[i] = getDistanceCost(reader, nodes.get(i)); } // Add cost/node pairs to a TreeMap to sort - TreeMap> tree = new TreeMap>(); + NavigableMap> tree = new TreeMap<>(); for (int i = 0; i < activeLen; i++) { int cost = costs[i]; - Node node = nodes.get(i); - List list = tree.get(cost); - if (list == null) { - list = Lists.newArrayListWithExpectedSize(1); - tree.put(cost, list); - } - list.add(node); + N node = nodes.get(i); + tree.computeIfAbsent(cost, k -> Lists.newArrayListWithExpectedSize(1)) + .add(node); } - List ret = new ArrayList<>(); - for (List list: tree.values()) { + List ret = new ArrayList<>(); + for (List list : tree.values()) { if (list != null) { shuffleOperation.accept(list); - for (Node n: list) { - ret.add(n); - } + ret.addAll(list); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index d2d8d0a63a0..94ab5fa6794 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.client.BlockID; @@ -353,17 +352,15 @@ public List sortDatanodes(List nodes, if (client == null) { client = getOtherNode(clientMachine); } - List nodeList = new ArrayList<>(); + List nodeList = new ArrayList<>(); nodes.forEach(uuid -> { DatanodeDetails node = nodeManager.getNodeByUuid(uuid); if (node != null) { nodeList.add(node); } }); - List sortedNodeList = scm.getClusterMap() + return scm.getClusterMap() .sortByDistanceCost(client, nodeList, nodeList.size()); - return sortedNodeList.stream().map(r -> (DatanodeDetails) r).collect( - Collectors.toList()); } catch (Exception ex) { auditSuccess = false; AUDIT.logReadFailure( From bf12f4e6bfb889ed5fb9ab02c7ffb0d8c2623ded Mon Sep 17 00:00:00 2001 From: Arafat2198 <98023601+ArafatKhan2198@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:54:41 +0530 Subject: [PATCH 30/51] HDDS-9766. Intermittent AlreadyClosedException in TestCommitWatcher.testReleaseBuffersOnException. (#5700) --- .../org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index ede67e55ee6..3e648fd8165 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -141,8 +141,8 @@ public void init() throws Exception { conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(7) - .setTotalPipelineNumLimit(10) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(3) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) From 38dc52478ea2de57f71dcbde433f69dd24a01a4a Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 29 Nov 2023 14:05:58 -0800 Subject: [PATCH 31/51] HDDS-9360. Throw an IOException if continuation token for snapshot diff is more than the total diff entries (#5697) --- .../ozone/snapshot/SnapshotDiffReportOzone.java | 17 ++++++++++------- .../ozone/om/snapshot/TestOmSnapshot.java | 8 ++++++++ .../ozone/om/snapshot/SnapshotDiffManager.java | 11 ++++++----- .../om/snapshot/TestSnapshotDiffManager.java | 4 ++-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java index aec0c6d12cc..3d14e266daa 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java @@ -101,13 +101,16 @@ public String toString() { .append(" and snapshot: ") .append(getLaterSnapshotName()) .append(LINE_SEPARATOR); - for (DiffReportEntry entry : getDiffList()) { - str.append(entry.toString()).append(LINE_SEPARATOR); - } - if (StringUtils.isNotEmpty(token)) { - str.append("Next token: ") - .append(token) - .append(LINE_SEPARATOR); + if (!getDiffList().isEmpty()) { + for (DiffReportEntry entry : getDiffList()) { + str.append(entry.toString()).append(LINE_SEPARATOR); + } + if (StringUtils.isNotEmpty(token)) { + str.append("Next token: ") + .append(token); + } + } else { + str.append("No diff or no more diff for the request parameters."); } return str.toString(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index ff444a88ceb..7e9fe787df6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -1429,6 +1429,14 @@ public void testSnapDiff() throws Exception { SnapshotDiffReport.DiffType.MODIFY, key3) ); assertEquals(expectedDiffList, diff5.getDiffList()); + + IOException ioException = assertThrows(IOException.class, + () -> store.snapshotDiff(volume, bucket, snap6, + snap7, "3", 0, forceFullSnapshotDiff, disableNativeDiff)); + assertThat(ioException.getMessage(), containsString("Index (given: 3) " + + "should be a number >= 0 and < totalDiffEntries: 2. Page size " + + "(given: 1000) should be a positive number > 0.")); + } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 651ed06cbe1..c34594cffe8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -569,11 +569,12 @@ SnapshotDiffReportOzone createPageResponse( final int index, final int pageSize ) throws IOException { - if (index < 0 || pageSize <= 0) { - throw new IllegalArgumentException(String.format( - "Index should be a number >= 0. Given index %d. Page size " + - "should be a positive number > 0. Given page size is %d", - index, pageSize)); + if (index < 0 || index > snapDiffJob.getTotalDiffEntries() + || pageSize <= 0) { + throw new IOException(String.format( + "Index (given: %d) should be a number >= 0 and < totalDiffEntries: " + + "%d. Page size (given: %d) should be a positive number > 0.", + index, snapDiffJob.getTotalDiffEntries(), pageSize)); } OFSPath path = getSnapshotRootPath(volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 8d8921ddff8..41b6f73d8cb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -893,7 +893,7 @@ private DiffReportEntry getTestDiffEntry(String jobId, * objectId Map of diff keys to be checked with their corresponding key names. */ @ParameterizedTest - @CsvSource({"0,10,1000", "1,10,8", "1000,1000,10", "-1,1000,10000", + @CsvSource({"0,10,1000", "1,10,8", "10,1000,10", "-1,1000,10000", "1,0,1000", "1,-1,1000"}) public void testCreatePageResponse(int startIdx, int pageSize, @@ -933,7 +933,7 @@ public void testCreatePageResponse(int startIdx, codecRegistry.asRawData(snapshotDiffJob2)); if (pageSize <= 0 || startIdx < 0) { - Assertions.assertThrows(IllegalArgumentException.class, + Assertions.assertThrows(IOException.class, () -> snapshotDiffManager.createPageResponse(snapshotDiffJob, "vol", "buck", "fs", "ts", startIdx, pageSize)); return; From cff506b1b23b0b3425af82b6c63b8c679cc3b774 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 30 Nov 2023 07:47:46 +0100 Subject: [PATCH 32/51] HDDS-9703. Check that dependencies have acceptable license (#5623) --- .github/workflows/ci.yml | 31 ++++++++ .../dev-support/checks/license.exceptions | 22 ++++++ hadoop-ozone/dev-support/checks/license.sh | 75 +++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 hadoop-ozone/dev-support/checks/license.exceptions create mode 100755 hadoop-ozone/dev-support/checks/license.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 617d1c9e3df..7696ffa9251 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -312,6 +312,37 @@ jobs: name: dependency path: target/dependency continue-on-error: true + license: + needs: + - build-info + - build + runs-on: ubuntu-20.04 + timeout-minutes: 15 + if: needs.build-info.outputs.needs-dependency-check == 'true' + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Download Ozone repo + id: download-ozone-repo + uses: actions/download-artifact@v3 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + - name: Execute tests + run: | + hadoop-ozone/dev-support/checks/${{ github.job }}.sh + continue-on-error: true + - name: Summary of failures + run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt + if: ${{ !cancelled() }} + - name: Archive build results + uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ github.job }} + path: target/${{ github.job }} + continue-on-error: true acceptance: needs: - build-info diff --git a/hadoop-ozone/dev-support/checks/license.exceptions b/hadoop-ozone/dev-support/checks/license.exceptions new file mode 100644 index 00000000000..66f17fb670d --- /dev/null +++ b/hadoop-ozone/dev-support/checks/license.exceptions @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file lists dependencies with acceptable license that +# license-maven-plugin cannot find, or finds with unexpected license. + +com.google.re2j:re2j:1.1 BSD 3-Clause +javax.servlet:servlet-api:2.5 CDDL 1.1 +javax.servlet.jsp:jsp-api:2.1 CDDL 1.1 +org.codehaus.jettison:jettison:1.1 Apache License 2.0 diff --git a/hadoop-ozone/dev-support/checks/license.sh b/hadoop-ozone/dev-support/checks/license.sh new file mode 100755 index 00000000000..f2c978731b3 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/license.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script checks if all third-party dependencies have licenses we can use. +# Optionally accepts the aggregated third-party license list file to be checked. +# Otherwise it requires Ozone to be available from Maven repo (can be local), +# so that it can generate the license list. +# +# When adding a new dependency to Ozone with a license that fails to match: +# * verify that the license is allowed, ref: https://www.apache.org/legal/resolved.html +# * tweak the patterns to allow +# +# Items for which license-maven-plugin cannot find license (e.g. jettison, +# jsp-api) are output as "Unknown license". These dependencies should be +# filtered explicitly by adding them to the `license.exceptions` file, instead +# of allowing the generic "Unknown license". + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/license"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="${REPORT_DIR}/summary.txt" + +DEFAULT_SRC="target/generated-sources/license/THIRD-PARTY.txt" +src="${1:-${DEFAULT_SRC}}" + +if [[ ! -e ${src} ]]; then + MAVEN_OPTIONS="-B -fae -Dskip.npx -Dskip.installnpx --no-transfer-progress ${MAVEN_OPTIONS:-}" + mvn ${MAVEN_OPTIONS} license:aggregate-add-third-party | tee "${REPORT_DIR}/output.log" + src="${DEFAULT_SRC}" +fi + +L='Licen[cs]e' # sometimes misspelled + +# filter all allowed licenses; any remaining item indicates a possible problem +grep '(' ${src} \ + | grep -v -f <(grep -v -e '^#' -e '^$' "${DIR}"/license.exceptions | cut -f1 -d' ') \ + | ( grep -i -v \ + -e "Apache ${L}" -e "Apache Software ${L}" -e "Apache v2" -e "Apache.2" \ + -e "Bouncy Castle ${L}" \ + -e "(BSD)" -e "(The BSD ${L})" -e "(BSD.[23]" -e "\.Clause.\" \ + -e "(CDDL\>" -e ' CDDL '\ + -e "(EDL\>" -e "Eclipse Distribution ${L}" \ + -e "(EPL\>" -e "Eclipse Public ${L}" \ + -e "(MIT)" -e "\" \ + -e "New BSD ${L}" \ + -e "Public Domain" \ + -e "Revised BSD\>" \ + || true ) \ + | sort -u \ + | tee "${REPORT_FILE}" + +wc -l "${REPORT_FILE}" | awk '{ print $1 }' > "${REPORT_DIR}/failures" + +if [[ -s "${REPORT_FILE}" ]]; then + exit 1 +fi From c656504b871bd0004b29aff0bb51698c90730840 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 30 Nov 2023 09:20:41 +0100 Subject: [PATCH 33/51] HDDS-9772. Avoid recreating typesafe config objects unnecessarily (#5690) --- .../server/ratis/XceiverServerRatis.java | 41 +++++++++---------- ...ocationProtocolClientSideTranslatorPB.java | 2 +- ...SCMBlockLocationFailoverProxyProvider.java | 28 ++++++------- .../hdds/scm/block/BlockManagerImpl.java | 5 ++- .../scm/block/SCMBlockDeletingService.java | 15 +++---- .../scm/server/StorageContainerManager.java | 2 +- 6 files changed, 44 insertions(+), 49 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 2a57a32b39c..0f8f7d4eccd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -136,22 +136,24 @@ private static long nextCallId() { private final List chunkExecutors; private final ContainerDispatcher dispatcher; private final ContainerController containerController; - private ClientId clientId = ClientId.randomId(); + private final ClientId clientId = ClientId.randomId(); private final StateContext context; - private long nodeFailureTimeoutMs; + private final long nodeFailureTimeoutMs; private boolean isStarted = false; - private DatanodeDetails datanodeDetails; + private final DatanodeDetails datanodeDetails; private final ConfigurationSource conf; // TODO: Remove the gids set when Ratis supports an api to query active // pipelines private final Set raftGids = ConcurrentHashMap.newKeySet(); private final RaftPeerId raftPeerId; // pipelines for which I am the leader - private Map groupLeaderMap = new ConcurrentHashMap<>(); + private final Map groupLeaderMap = + new ConcurrentHashMap<>(); // Timeout used while calling submitRequest directly. - private long requestTimeout; - private boolean shouldDeleteRatisLogDirectory; - private boolean streamEnable; + private final long requestTimeout; + private final boolean shouldDeleteRatisLogDirectory; + private final boolean streamEnable; + private final DatanodeRatisServerConfig ratisServerConfig; private XceiverServerRatis(DatanodeDetails dd, ContainerDispatcher dispatcher, ContainerController containerController, @@ -160,6 +162,7 @@ private XceiverServerRatis(DatanodeDetails dd, this.conf = conf; Objects.requireNonNull(dd, "id == null"); datanodeDetails = dd; + ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class); assignPorts(); this.streamEnable = conf.getBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, @@ -171,12 +174,9 @@ private XceiverServerRatis(DatanodeDetails dd, this.raftPeerId = RatisHelper.toRaftPeerId(dd); String threadNamePrefix = datanodeDetails.threadNamePrefix(); chunkExecutors = createChunkExecutors(conf, threadNamePrefix); - nodeFailureTimeoutMs = - conf.getObject(DatanodeRatisServerConfig.class) - .getFollowerSlownessTimeout(); + nodeFailureTimeoutMs = ratisServerConfig.getFollowerSlownessTimeout(); shouldDeleteRatisLogDirectory = - conf.getObject(DatanodeRatisServerConfig.class) - .shouldDeleteRatisLogDirectory(); + ratisServerConfig.shouldDeleteRatisLogDirectory(); this.server = RaftServer.newBuilder().setServerId(raftPeerId) @@ -237,13 +237,10 @@ private void setUpRatisStream(RaftProperties properties) { RatisHelper.enableNettyStreaming(properties); NettyConfigKeys.DataStream.setPort(properties, dataStreamPort); int dataStreamAsyncRequestThreadPoolSize = - conf.getObject(DatanodeRatisServerConfig.class) - .getStreamRequestThreads(); + ratisServerConfig.getStreamRequestThreads(); RaftServerConfigKeys.DataStream.setAsyncRequestThreadPoolSize(properties, dataStreamAsyncRequestThreadPoolSize); - int dataStreamClientPoolSize = - conf.getObject(DatanodeRatisServerConfig.class) - .getClientPoolSize(); + int dataStreamClientPoolSize = ratisServerConfig.getClientPoolSize(); RaftServerConfigKeys.DataStream.setClientPoolSize(properties, dataStreamClientPoolSize); } @@ -309,13 +306,13 @@ public RaftProperties newRaftProperties() { // Disable the pre vote feature in Ratis RaftServerConfigKeys.LeaderElection.setPreVote(properties, - conf.getObject(DatanodeRatisServerConfig.class).isPreVoteEnabled()); + ratisServerConfig.isPreVoteEnabled()); // Set the ratis storage directory Collection storageDirPaths = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf); List storageDirs = new ArrayList<>(storageDirPaths.size()); - storageDirPaths.stream().forEach(d -> storageDirs.add(new File(d))); + storageDirPaths.forEach(d -> storageDirs.add(new File(d))); RaftServerConfigKeys.setStorageDir(properties, storageDirs); @@ -621,16 +618,16 @@ private void processReply(RaftClientReply reply) throws IOException { @Override public void submitRequest(ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID) throws IOException { - RaftClientReply reply = null; Span span = TracingUtil .importAndCreateSpan( "XceiverServerRatis." + request.getCmdType().name(), request.getTraceID()); - try (Scope scope = GlobalTracer.get().activateSpan(span)) { + try (Scope ignored = GlobalTracer.get().activateSpan(span)) { RaftClientRequest raftClientRequest = createRaftClientRequest(request, pipelineID, RaftClientRequest.writeRequestType()); + RaftClientReply reply; try { reply = server.submitClientRequestAsync(raftClientRequest) .get(requestTimeout, TimeUnit.MILLISECONDS); @@ -896,7 +893,7 @@ public long getMinReplicatedIndex(PipelineID pipelineID) throws IOException { GroupInfoReply reply = getServer() .getGroupInfo(createGroupInfoRequest(pipelineID.getProtobuf())); minIndex = RatisHelper.getMinReplicatedIndex(reply.getCommitInfos()); - return minIndex == null ? -1 : minIndex.longValue(); + return minIndex == null ? -1 : minIndex; } public void notifyGroupRemove(RaftGroupId gid) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index eb19e46b7ac..3c20295bd6f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -94,7 +94,7 @@ public ScmBlockLocationProtocolClientSideTranslatorPB( this.failoverProxyProvider = proxyProvider; this.rpcProxy = (ScmBlockLocationProtocolPB) RetryProxy.create( ScmBlockLocationProtocolPB.class, failoverProxyProvider, - failoverProxyProvider.getSCMBlockLocationRetryPolicy(null)); + failoverProxyProvider.getSCMBlockLocationRetryPolicy()); } /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java index 5cff8888306..0c58e8cc510 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java @@ -56,9 +56,10 @@ public class SCMBlockLocationFailoverProxyProvider implements FailoverProxyProvider, Closeable { public static final Logger LOG = LoggerFactory.getLogger(SCMBlockLocationFailoverProxyProvider.class); + private final SCMClientConfig scmClientConfig; - private Map> scmProxies; - private Map scmProxyInfoMap; + private final Map> scmProxies; + private final Map scmProxyInfoMap; private List scmNodeIds; // As SCM Client is shared across threads, performFailOver() @@ -107,9 +108,9 @@ public SCMBlockLocationFailoverProxyProvider(ConfigurationSource conf) { this.currentProxyIndex = 0; currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex); - SCMClientConfig config = conf.getObject(SCMClientConfig.class); - this.maxRetryCount = config.getRetryCount(); - this.retryInterval = config.getRetryInterval(); + scmClientConfig = conf.getObject(SCMClientConfig.class); + this.maxRetryCount = scmClientConfig.getRetryCount(); + this.retryInterval = scmClientConfig.getRetryInterval(); LOG.info("Created block location fail-over proxy with {} nodes: {}", scmNodeIds.size(), scmProxyInfoMap.values()); @@ -146,15 +147,15 @@ public synchronized void changeCurrentProxy(String nodeId) { nextProxyIndex(); } - @VisibleForTesting - public synchronized String getCurrentProxySCMNodeId() { + private synchronized String getCurrentProxySCMNodeId() { return currentProxySCMNodeId; } @Override public synchronized ProxyInfo getProxy() { String currentProxyNodeId = getCurrentProxySCMNodeId(); - ProxyInfo currentProxyInfo = scmProxies.get(currentProxyNodeId); + ProxyInfo currentProxyInfo = + scmProxies.get(currentProxyNodeId); if (currentProxyInfo == null) { currentProxyInfo = createSCMProxy(currentProxyNodeId); } @@ -238,8 +239,8 @@ private synchronized void assignLeaderToNode(String newLeaderNodeId) { /** * Creates proxy object. */ - private ProxyInfo createSCMProxy(String nodeId) { - ProxyInfo proxyInfo; + private ProxyInfo createSCMProxy(String nodeId) { + ProxyInfo proxyInfo; SCMProxyInfo scmProxyInfo = scmProxyInfoMap.get(nodeId); InetSocketAddress address = scmProxyInfo.getAddress(); try { @@ -269,12 +270,12 @@ private ScmBlockLocationProtocolPB createSCMProxy( return RPC.getProtocolProxy(ScmBlockLocationProtocolPB.class, scmVersion, scmAddress, ugi, hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf), - (int)conf.getObject(SCMClientConfig.class).getRpcTimeOut(), + (int) scmClientConfig.getRpcTimeOut(), connectionRetryPolicy).getProxy(); } - public RetryPolicy getSCMBlockLocationRetryPolicy(String newLeader) { - RetryPolicy retryPolicy = new RetryPolicy() { + public RetryPolicy getSCMBlockLocationRetryPolicy() { + return new RetryPolicy() { @Override public RetryAction shouldRetry(Exception e, int retry, int failover, boolean b) { @@ -287,7 +288,6 @@ public RetryAction shouldRetry(Exception e, int retry, getRetryInterval()); } }; - return retryPolicy; } public synchronized int getCurrentProxyIndex() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java index 48830cafbd2..1260ea6a006 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; @@ -79,7 +80,7 @@ public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean { * @throws IOException */ public BlockManagerImpl(final ConfigurationSource conf, - final StorageContainerManager scm) + ScmConfig scmConfig, final StorageContainerManager scm) throws IOException { Objects.requireNonNull(scm, "SCM cannot be null"); this.scm = scm; @@ -108,7 +109,7 @@ public BlockManagerImpl(final ConfigurationSource conf, blockDeletingService = new SCMBlockDeletingService(deletedBlockLog, scm.getScmNodeManager(), scm.getEventQueue(), scm.getScmContext(), - scm.getSCMServiceManager(), conf, + scm.getSCMServiceManager(), conf, scmConfig, metrics, scm.getSystemClock(), scm.getReconfigurationHandler()); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java index 16bfed2bbaa..5fd889b7583 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java @@ -81,8 +81,7 @@ public class SCMBlockDeletingService extends BackgroundService private final SCMContext scmContext; private final ScmConfig scmConf; - private int blockDeleteLimitSize; - private ScmBlockDeletingServiceMetrics metrics; + private final ScmBlockDeletingServiceMetrics metrics; /** * SCMService related variables. @@ -99,11 +98,10 @@ public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog, NodeManager nodeManager, EventPublisher eventPublisher, SCMContext scmContext, SCMServiceManager serviceManager, ConfigurationSource conf, - ScmBlockDeletingServiceMetrics metrics, + ScmConfig scmConfig, ScmBlockDeletingServiceMetrics metrics, Clock clock, ReconfigurationHandler reconfigurationHandler) { super("SCMBlockDeletingService", - conf.getObject(ScmConfig.class) - .getBlockDeletionInterval().toMillis(), + scmConfig.getBlockDeletionInterval().toMillis(), TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, conf.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT, OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, @@ -119,11 +117,10 @@ public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog, this.eventPublisher = eventPublisher; this.scmContext = scmContext; this.metrics = metrics; - scmConf = conf.getObject(ScmConfig.class); - reconfigurationHandler.register(scmConf); - blockDeleteLimitSize = scmConf.getBlockDeletionLimit(); - Preconditions.checkArgument(blockDeleteLimitSize > 0, + scmConf = scmConfig; + Preconditions.checkArgument(scmConf.getBlockDeletionLimit() > 0, "Block deletion limit should be positive."); + reconfigurationHandler.register(scmConf); // register SCMBlockDeletingService to SCMServiceManager serviceManager.register(this); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index edba7bb6d34..59a533a665d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -806,7 +806,7 @@ private void initializeSystemManagers(OzoneConfiguration conf, if (configurator.getScmBlockManager() != null) { scmBlockManager = configurator.getScmBlockManager(); } else { - scmBlockManager = new BlockManagerImpl(conf, this); + scmBlockManager = new BlockManagerImpl(conf, scmConfig, this); } if (configurator.getReplicationManager() != null) { replicationManager = configurator.getReplicationManager(); From 728e1fe444cb44a30c4241a099a2d69b3817a791 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:52:17 +0100 Subject: [PATCH 34/51] HDDS-9504. Migrate some parameterized integration tests to JUnit5 (#5679) --- .../fs/ozone/TestOzoneFSBucketLayout.java | 190 +++----- .../apache/hadoop/ozone/MiniOzoneCluster.java | 6 +- ...estOzoneRpcClientWithKeyLatestVersion.java | 229 ++++----- .../ozone/client/rpc/TestReadRetries.java | 302 ++++-------- .../client/rpc/read/TestChunkInputStream.java | 101 ++-- .../client/rpc/read/TestInputStreamBase.java | 166 +------ .../client/rpc/read/TestKeyInputStream.java | 298 ++++++------ .../container/TestContainerReplication.java | 170 +++---- .../ozoneimpl/TestOzoneContainerWithTLS.java | 298 +++++------- .../ozoneimpl/TestSecureOzoneContainer.java | 95 ++-- ...groundContainerDataScannerIntegration.java | 56 +-- ...ndContainerMetadataScannerIntegration.java | 64 ++- ...stContainerScannerIntegrationAbstract.java | 61 +-- ...DemandContainerDataScannerIntegration.java | 50 +- ...estDatanodeHddsVolumeFailureDetection.java | 459 ++++++++---------- .../apache/hadoop/ozone/om/TestBucket.java | 140 ++++++ .../ozone/om/TestOMBucketLayoutUpgrade.java | 241 ++++----- .../ozone/om/TestOMUpgradeFinalization.java | 224 +++------ 18 files changed, 1310 insertions(+), 1840 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucket.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index a23bebdf259..d9d347dac3e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs.ozone; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; @@ -27,19 +26,19 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.junit.BeforeClass; -import org.junit.AfterClass; -import org.junit.Test; -import org.junit.Assert; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterAll; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_ROOT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -47,8 +46,9 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Arrays; import java.util.Collection; @@ -56,63 +56,50 @@ /** * Ozone file system tests to validate default bucket layout configuration * and behaviour. - * TODO: Refactor this and TestOzoneFileSystem to reduce duplication. + * TODO: merge with some other test */ -@RunWith(Parameterized.class) -public class TestOzoneFSBucketLayout { - - private static String defaultBucketLayout; - private static MiniOzoneCluster cluster = null; - private static ObjectStore objectStore; - private static OzoneClient client; - private BasicRootedOzoneClientAdapterImpl adapter; - private static String rootPath; - private static String volumeName; - private static Path volumePath; - - private static final String INVALID_CONFIG = "INVALID"; - private static final Map ERROR_MAP = new HashMap<>(); +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestOzoneFSBucketLayout { + + private MiniOzoneCluster cluster; + private ObjectStore objectStore; + private OzoneClient client; + private String rootPath; + private String volumeName; - private static final Logger LOG = - LoggerFactory.getLogger(TestOzoneFSBucketLayout.class); + private static final String UNKNOWN_LAYOUT = "INVALID"; + private static final Map ERROR_MAP = new HashMap<>(); // Initialize error map. static { ERROR_MAP.put(BucketLayout.OBJECT_STORE.name(), "Buckets created with OBJECT_STORE layout do not support file " + "system semantics."); - ERROR_MAP.put(INVALID_CONFIG, "Unsupported value provided for " + + ERROR_MAP.put(UNKNOWN_LAYOUT, "Unsupported value provided for " + OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); } - @Parameterized.Parameters - public static Collection data() { + static Collection validDefaultBucketLayouts() { return Arrays.asList( // Empty Config "", - // Invalid Config - INVALID_CONFIG, - // Unsupported Bucket Layout for OFS - BucketLayout.OBJECT_STORE.name(), // Supported bucket layouts. BucketLayout.FILE_SYSTEM_OPTIMIZED.name(), BucketLayout.LEGACY.name() ); } - public TestOzoneFSBucketLayout(String bucketLayout) { - // Ignored. Actual init done in initParam(). - // This empty constructor is still required to avoid argument exception. - } - - @Parameterized.BeforeParam - public static void initDefaultLayout(String bucketLayout) { - defaultBucketLayout = bucketLayout; - LOG.info("Default bucket layout: {}", defaultBucketLayout); + static Collection invalidDefaultBucketLayouts() { + return Arrays.asList( + // Invalid Config + UNKNOWN_LAYOUT, + // Unsupported Bucket Layout for OFS + BucketLayout.OBJECT_STORE.name() + ); } - @BeforeClass - public static void initCluster() throws Exception { + @BeforeAll + void initCluster() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) @@ -122,89 +109,64 @@ public static void initCluster() throws Exception { objectStore = client.getObjectStore(); rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); - - // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) - volumeName = - TestDataUtil.createVolumeAndBucket(client) - .getVolumeName(); - volumePath = new Path(OZONE_URI_DELIMITER, volumeName); + volumeName = TestDataUtil.createVolumeAndBucket(client).getVolumeName(); } - @AfterClass - public static void teardown() throws IOException { + @AfterAll + void teardown() throws IOException { IOUtils.closeQuietly(client); - // Tear down the cluster after EACH set of parameters if (cluster != null) { cluster.shutdown(); } } - @Test - public void testFileSystemBucketLayoutConfiguration() throws IOException { - OzoneConfiguration conf = new OzoneConfiguration(); - - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setFsDefaultBucketLayout(defaultBucketLayout); - - conf.setFromObject(clientConfig); - - // Set the fs.defaultFS and start the filesystem - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + @ParameterizedTest + @MethodSource("invalidDefaultBucketLayouts") + void fileSystemWithUnsupportedDefaultBucketLayout(String layout) { + OzoneConfiguration conf = configWithDefaultBucketLayout(layout); - // In case OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT is set to OBS, - // FS initialization should fail. - - if (ERROR_MAP.containsKey(defaultBucketLayout)) { - try { - FileSystem.newInstance(conf); - Assert.fail("File System initialization should fail in case " + - " of invalid configuration of " + - OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT); - } catch (OMException oe) { - Assert.assertTrue( - oe.getMessage().contains(ERROR_MAP.get(defaultBucketLayout))); - return; - } + OMException e = assertThrows(OMException.class, + () -> FileSystem.newInstance(conf)); + assertThat(e.getMessage(), + containsString(ERROR_MAP.get(layout))); + } + @ParameterizedTest + @MethodSource("validDefaultBucketLayouts") + void fileSystemWithValidBucketLayout(String layout) throws IOException { + OzoneConfiguration conf = configWithDefaultBucketLayout(layout); + + try (FileSystem fs = FileSystem.newInstance(conf)) { + // Create a new directory, which in turn creates a new bucket. + String bucketName = getBucketName(layout); + fs.mkdirs(new Path(new Path(OZONE_ROOT, volumeName), bucketName)); + + // Make sure the bucket layout of created bucket matches the config. + OzoneBucket bucketInfo = + objectStore.getClientProxy().getBucketDetails(volumeName, bucketName); + + String expectedLayout = layout.isEmpty() + ? OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT + : layout; + assertEquals(expectedLayout, bucketInfo.getBucketLayout().name()); } + } - // initialize FS and adapter. - FileSystem fs = FileSystem.newInstance(conf); - RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fs; - adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter(); - - // Create a new directory, which in turn creates a new bucket. - Path root = new Path("/" + volumeName); - - String bucketName = getBucketName(); - Path dir1 = new Path(root, bucketName); - - adapter.createDirectory(dir1.toString()); - - // Make sure the bucket layout of created bucket matches the config. - OzoneBucket bucketInfo = - objectStore.getClientProxy().getBucketDetails(volumeName, bucketName); - if (StringUtils.isNotBlank(defaultBucketLayout)) { - Assert.assertEquals(defaultBucketLayout, - bucketInfo.getBucketLayout().name()); - } else { - Assert.assertEquals(OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT, - bucketInfo.getBucketLayout().name()); - } + private String getBucketName(String layout) { + String bucketSuffix = layout.isEmpty() + ? "empty" + : layout.toLowerCase().replaceAll("_", "-"); - // cleanup - IOUtils.closeQuietly(fs); + return "bucket-" + bucketSuffix; } - private String getBucketName() { - String bucketSuffix; - if (StringUtils.isNotBlank(defaultBucketLayout)) { - bucketSuffix = defaultBucketLayout - .toLowerCase() - .replaceAll("_", "-"); - } else { - bucketSuffix = "empty"; - } + private OzoneConfiguration configWithDefaultBucketLayout(String layout) { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - return "bucket-" + bucketSuffix; + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setFsDefaultBucketLayout(layout); + conf.setFromObject(clientConfig); + return conf; } + } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index a1696f049d4..91481dbaeca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -45,7 +45,7 @@ /** * Interface used for MiniOzoneClusters. */ -public interface MiniOzoneCluster { +public interface MiniOzoneCluster extends AutoCloseable { /** * Returns the Builder to construct MiniOzoneCluster. @@ -260,6 +260,10 @@ void restartHddsDatanode(DatanodeDetails dn, boolean waitForDatanode) */ void shutdown(); + default void close() { + shutdown(); + } + /** * Stop the MiniOzoneCluster without any cleanup. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java index ba584d5048d..fd32698eec2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,11 @@ package org.apache.hadoop.ozone.client.rpc; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; @@ -28,183 +30,126 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collection; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.HashMap; import java.util.List; import java.util.UUID; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Main purpose of this test is with OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION * set/unset key create/read works properly or not for buckets - * with/with out versioning. + * with/without versioning. + * TODO: can be merged with other test class */ -@RunWith(Parameterized.class) -public class TestOzoneRpcClientWithKeyLatestVersion { - - private static MiniOzoneCluster cluster; - - private ObjectStore objectStore; - - private OzoneClient ozClient; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestOzoneRpcClientWithKeyLatestVersion { - private final boolean getLatestVersion; - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(true, false); - } - - public TestOzoneRpcClientWithKeyLatestVersion(boolean latestVersion) { - getLatestVersion = latestVersion; - } + private MiniOzoneCluster cluster; - @BeforeClass - public static void setup() throws Exception { + @BeforeAll + void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .setScmId(UUID.randomUUID().toString()) - .setClusterId(UUID.randomUUID().toString()) .build(); cluster.waitForClusterToBeReady(); } - @AfterClass - public static void tearDown() throws Exception { + @AfterAll + void tearDown() { if (cluster != null) { cluster.shutdown(); } } - @Before - public void setupClient() throws Exception { - OzoneConfiguration conf = cluster.getConf(); - conf.setBoolean(OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION, getLatestVersion); - ozClient = OzoneClientFactory.getRpcClient(conf); - objectStore = ozClient.getObjectStore(); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testWithGetLatestVersion(boolean getLatestVersionOnly) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(cluster.getConf()); + conf.setBoolean(OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION, + getLatestVersionOnly); + + try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) { + String volumeName = UUID.randomUUID().toString(); + ObjectStore objectStore = client.getObjectStore(); + objectStore.createVolume(volumeName); + OzoneVolume volume = objectStore.getVolume(volumeName); + + for (boolean versioning : new boolean[] {false, true}) { + String bucketName = UUID.randomUUID().toString(); + volume.createBucket(bucketName, + BucketArgs.newBuilder() + .setVersioning(versioning) + .build()); + + OzoneBucket bucket = volume.getBucket(bucketName); + String keyName = UUID.randomUUID().toString(); + byte[] content = RandomUtils.nextBytes(128); + int versions = RandomUtils.nextInt(2, 5); + + createAndOverwriteKey(bucket, keyName, versions, content); + + assertKeyContent(bucket, keyName, content); + + int expectedVersionCount = + versioning && !getLatestVersionOnly ? versions : 1; + assertListStatus(bucket, keyName, expectedVersionCount); + } + } } - @After - public void closeClient() throws Exception { - if (ozClient != null) { - ozClient.close(); + /** Repeatedly write {@code key}, first some random data, then + * {@code content}. */ + private void createAndOverwriteKey(OzoneBucket bucket, String key, + int versions, byte[] content) throws IOException { + ReplicationConfig replication = RatisReplicationConfig.getInstance(THREE); + for (int i = 1; i < versions; i++) { + writeKey(bucket, key, RandomUtils.nextBytes(content.length), replication); } + // overwrite it + writeKey(bucket, key, content, replication); } - - @Test - public void testWithGetLatestVersion() throws Exception { - testOverrideAndReadKey(); + private static void writeKey(OzoneBucket bucket, String key, byte[] content, + ReplicationConfig replication) throws IOException { + try (OutputStream out = bucket.createKey(key, content.length, replication, + new HashMap<>())) { + out.write(content); + } } - - - public void testOverrideAndReadKey() throws Exception { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); - - // Test checks key override and read are working with - // bucket versioning false. - String value = "sample value"; - createRequiredForVersioningTest(volumeName, bucketName, keyName, - false, value); - - // read key and test - testReadKey(volumeName, bucketName, keyName, value); - - testListStatus(volumeName, bucketName, keyName, false); - - - // Bucket versioning turned on - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); - keyName = UUID.randomUUID().toString(); - - // Test checks key override and read are working with - // bucket versioning true. - createRequiredForVersioningTest(volumeName, bucketName, keyName, - true, value); - - // read key and test - testReadKey(volumeName, bucketName, keyName, value); - - - testListStatus(volumeName, bucketName, keyName, true); + public static void assertKeyContent(OzoneBucket bucket, String key, + byte[] expected) throws Exception { + try (InputStream in = bucket.readKey(key)) { + assertArrayEquals(expected, IOUtils.readFully(in, expected.length)); + } } - private void createRequiredForVersioningTest(String volumeName, - String bucketName, String keyName, boolean versioning, - String value) throws Exception { - - ReplicationConfig replicationConfig = ReplicationConfig - .fromProtoTypeAndFactor(RATIS, HddsProtos.ReplicationFactor.THREE); + private void assertListStatus(OzoneBucket bucket, String keyName, + int expectedVersionCount) throws Exception { + List files = bucket.listStatus(keyName, false, "", 1); - objectStore.createVolume(volumeName); - OzoneVolume volume = objectStore.getVolume(volumeName); + assertNotNull(files); + assertEquals(1, files.size()); - // Bucket created with versioning false. - volume.createBucket(bucketName, - BucketArgs.newBuilder().setVersioning(versioning).build()); - OzoneBucket bucket = volume.getBucket(bucketName); - - OzoneOutputStream out = bucket.createKey(keyName, - value.getBytes(UTF_8).length, replicationConfig, new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - - // Override key - out = bucket.createKey(keyName, - value.getBytes(UTF_8).length, replicationConfig, new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - } - - private void testReadKey(String volumeName, String bucketName, - String keyName, String value) throws Exception { - OzoneVolume volume = objectStore.getVolume(volumeName); - OzoneBucket ozoneBucket = volume.getBucket(bucketName); - byte[] fileContent = new byte[value.getBytes(UTF_8).length]; - try (OzoneInputStream is = ozoneBucket.readKey(keyName)) { - is.read(fileContent); - } - Assert.assertEquals(value, new String(fileContent, UTF_8)); - } - - private void testListStatus(String volumeName, String bucketName, - String keyName, boolean versioning) throws Exception { - OzoneVolume volume = objectStore.getVolume(volumeName); - OzoneBucket ozoneBucket = volume.getBucket(bucketName); - List ozoneFileStatusList = ozoneBucket.listStatus(keyName, - false, "", 1); - Assert.assertNotNull(ozoneFileStatusList); - Assert.assertEquals(1, ozoneFileStatusList.size()); - if (!getLatestVersion && versioning) { - Assert.assertEquals(2, ozoneFileStatusList.get(0).getKeyInfo() - .getKeyLocationVersions().size()); - } else { - Assert.assertEquals(1, ozoneFileStatusList.get(0).getKeyInfo() - .getKeyLocationVersions().size()); - } + List versions = files.get(0).getKeyInfo().getKeyLocationVersions(); + assertEquals(expectedVersionCount, versions.size()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java index fbcc177be30..5df762d6fd4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -18,243 +18,139 @@ package org.apache.hadoop.ozone.client.rpc; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; +import java.io.OutputStream; import java.util.HashMap; import java.util.List; import java.util.UUID; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.XceiverClientFactory; -import org.apache.hadoop.hdds.scm.XceiverClientRatis; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneKeyLocation; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.KeyOutputStream; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.junit.After; -import org.junit.Assert; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.fail; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.client.rpc.TestOzoneRpcClientWithKeyLatestVersion.assertKeyContent; +import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.configureFSOptimizedPaths; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.rules.ExpectedException; - -/** - * Test read retries from multiple nodes in the pipeline. - */ -@RunWith(Parameterized.class) -public class TestReadRetries { - - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private MiniOzoneCluster cluster = null; - private OzoneClient ozClient = null; - private ObjectStore store = null; - private OzoneManager ozoneManager; - private StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - - private static final String SCM_ID = UUID.randomUUID().toString(); - private String bucketLayout; - - public TestReadRetries(String bucketLayout) { - this.bucketLayout = bucketLayout; - } - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[]{OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT}, - new Object[]{OMConfigKeys. - OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED}); - } +@Timeout(300) +class TestReadRetries { /** - * Create a MiniOzoneCluster for testing. - * @throws Exception + * Test read retries from multiple nodes in the pipeline. */ - @Before - public void init() throws Exception { + @Test + void testPutKeyAndGetKeyThreeNodes() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); - OMRequestTestUtils.configureFSOptimizedPaths(conf, - true, BucketLayout.fromString(bucketLayout)); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .setScmId(SCM_ID) - .build(); - cluster.waitForClusterToBeReady(); - cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.THREE, - 180000); - ozClient = OzoneClientFactory.getRpcClient(conf); - store = ozClient.getObjectStore(); - storageContainerLocationClient = - cluster.getStorageContainerLocationClient(); - ozoneManager = cluster.getOzoneManager(); - } - - - /** - * Close OzoneClient and shutdown MiniOzoneCluster. - */ - @After - public void shutdown() throws IOException { - if (ozClient != null) { - ozClient.close(); - } - - if (storageContainerLocationClient != null) { - storageContainerLocationClient.close(); - } - - if (cluster != null) { - cluster.shutdown(); + configureFSOptimizedPaths(conf, true, BucketLayout.FILE_SYSTEM_OPTIMIZED); + + try (MiniOzoneCluster cluster = newCluster(conf)) { + cluster.waitForClusterToBeReady(); + cluster.waitForPipelineTobeReady(THREE, 180000); + + try (OzoneClient client = cluster.newClient()) { + ObjectStore store = client.getObjectStore(); + + String volumeName = UUID.randomUUID().toString(); + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + + String bucketName = UUID.randomUUID().toString(); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + String keyName = "a/b/c/" + UUID.randomUUID(); + byte[] content = RandomUtils.nextBytes(128); + try (OutputStream out = bucket.createKey(keyName, content.length, + RatisReplicationConfig.getInstance(THREE), new HashMap<>())) { + out.write(content); + } + + // First, confirm the key info from the client matches the info in OM. + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .build(); + OmKeyLocationInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs) + .getKeyLocationVersions().get(0) + .getBlocksLatestVersionOnly().get(0); + long containerID = keyInfo.getContainerID(); + + OzoneKeyDetails keyDetails = bucket.getKey(keyName); + assertEquals(keyName, keyDetails.getName()); + + List keyLocations = keyDetails.getOzoneKeyLocations(); + assertEquals(1, keyLocations.size()); + assertEquals(containerID, keyLocations.get(0).getContainerID()); + assertEquals(keyInfo.getLocalID(), keyLocations.get(0).getLocalID()); + + // Make sure that the data size matched. + assertEquals(content.length, keyLocations.get(0).getLength()); + + StorageContainerManager scm = cluster.getStorageContainerManager(); + ContainerInfo container = scm.getContainerManager() + .getContainer(ContainerID.valueOf(containerID)); + Pipeline pipeline = scm.getPipelineManager() + .getPipeline(container.getPipelineID()); + List datanodes = pipeline.getNodes(); + assertEquals(3, datanodes.size()); + + // shutdown the datanode + cluster.shutdownHddsDatanode(datanodes.get(0)); + // try to read, this should be successful + assertKeyContent(bucket, keyName, content); + + // shutdown the second datanode + cluster.shutdownHddsDatanode(datanodes.get(1)); + // we still should be able to read + assertKeyContent(bucket, keyName, content); + + // shutdown the 3rd datanode + cluster.shutdownHddsDatanode(datanodes.get(2)); + // no longer can read it + assertThrows(IOException.class, + () -> assertKeyContent(bucket, keyName, content)); + + // read intermediate directory + verifyIntermediateDir(bucket, "a/b/c"); + } } } - - @Test - public void testPutKeyAndGetKeyThreeNodes() - throws Exception { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - - String value = "sample value"; - store.createVolume(volumeName); - OzoneVolume volume = store.getVolume(volumeName); - volume.createBucket(bucketName); - OzoneBucket bucket = volume.getBucket(bucketName); - - String keyName = "a/b/c/" + UUID.randomUUID().toString(); - - OzoneOutputStream out = bucket - .createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS, - ReplicationFactor.THREE, new HashMap<>()); - KeyOutputStream groupOutputStream = - (KeyOutputStream) out.getOutputStream(); - XceiverClientFactory factory = groupOutputStream.getXceiverClientFactory(); - out.write(value.getBytes(UTF_8)); - out.close(); - // First, confirm the key info from the client matches the info in OM. - OmKeyArgs.Builder builder = new OmKeyArgs.Builder(); - builder.setVolumeName(volumeName).setBucketName(bucketName) - .setKeyName(keyName); - OmKeyLocationInfo keyInfo = ozoneManager.lookupKey(builder.build()). - getKeyLocationVersions().get(0).getBlocksLatestVersionOnly().get(0); - long containerID = keyInfo.getContainerID(); - long localID = keyInfo.getLocalID(); - OzoneKeyDetails keyDetails = bucket.getKey(keyName); - Assert.assertEquals(keyName, keyDetails.getName()); - - List keyLocations = keyDetails.getOzoneKeyLocations(); - Assert.assertEquals(1, keyLocations.size()); - Assert.assertEquals(containerID, keyLocations.get(0).getContainerID()); - Assert.assertEquals(localID, keyLocations.get(0).getLocalID()); - - // Make sure that the data size matched. - Assert.assertEquals(value.getBytes(UTF_8).length, - keyLocations.get(0).getLength()); - - ContainerInfo container = cluster.getStorageContainerManager() - .getContainerManager().getContainer(ContainerID.valueOf(containerID)); - Pipeline pipeline = cluster.getStorageContainerManager() - .getPipelineManager().getPipeline(container.getPipelineID()); - List datanodes = pipeline.getNodes(); - - DatanodeDetails datanodeDetails = datanodes.get(0); - Assert.assertNotNull(datanodeDetails); - - XceiverClientSpi clientSpi = factory.acquireClient(pipeline); - Assert.assertTrue(clientSpi instanceof XceiverClientRatis); - XceiverClientRatis ratisClient = (XceiverClientRatis)clientSpi; - - ratisClient.watchForCommit(keyInfo.getBlockCommitSequenceId()); - // shutdown the datanode - cluster.shutdownHddsDatanode(datanodeDetails); - // try to read, this should be successful - readKey(bucket, keyName, value); - - // read intermediate directory - verifyIntermediateDir(bucket, "a/b/c"); - - // shutdown the second datanode - datanodeDetails = datanodes.get(1); - cluster.shutdownHddsDatanode(datanodeDetails); - - // we still should be able to read via Standalone protocol - // try to read - readKey(bucket, keyName, value); - - // shutdown the 3rd datanode - datanodeDetails = datanodes.get(2); - cluster.shutdownHddsDatanode(datanodeDetails); - try { - // try to read - readKey(bucket, keyName, value); - fail("Expected exception not thrown"); - } catch (IOException e) { - // it should throw an ioException as none of the servers - // are available - } - factory.releaseClient(clientSpi, false); - } - - private void verifyIntermediateDir(OzoneBucket bucket, String dir) + private static MiniOzoneCluster newCluster(OzoneConfiguration conf) throws IOException { - OzoneFileStatus fileStatus = bucket.getFileStatus(dir); - Assert.assertTrue(fileStatus.isDirectory()); - Assert.assertEquals(dir, fileStatus.getTrimmedName()); + return MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .build(); } - private void readKey(OzoneBucket bucket, String keyName, String data) + private static void verifyIntermediateDir(OzoneBucket bucket, String dir) throws IOException { - OzoneKey key = bucket.getKey(keyName); - Assert.assertEquals(keyName, key.getName()); - OzoneInputStream is = bucket.readKey(keyName); - byte[] fileContent = new byte[data.getBytes(UTF_8).length]; - is.read(fileContent); - is.close(); + OzoneFileStatus fileStatus = bucket.getFileStatus(dir); + assertTrue(fileStatus.isDirectory()); + assertEquals(dir, fileStatus.getTrimmedName()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java index cc675ecce0c..cc7375c0347 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java @@ -19,32 +19,50 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import org.apache.hadoop.hdds.scm.storage.BlockInputStream; import org.apache.hadoop.hdds.scm.storage.ChunkInputStream; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.KeyInputStream; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; -import org.junit.Assert; -import org.junit.Test; +import org.apache.hadoop.ozone.om.TestBucket; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; /** * Tests {@link ChunkInputStream}. */ -public class TestChunkInputStream extends TestInputStreamBase { +class TestChunkInputStream extends TestInputStreamBase { - public TestChunkInputStream(ContainerLayoutVersion layout) { - super(layout); + private static List layouts() { + return ContainerLayoutVersion.getAllVersions(); } /** * Run the tests as a single test method to avoid needing a new mini-cluster * for each test. */ - @Test - public void testAll() throws Exception { - testChunkReadBuffers(); - testBufferRelease(); - testCloseReleasesBuffers(); + @ParameterizedTest + @MethodSource("layouts") + void testAll(ContainerLayoutVersion layout) throws Exception { + try (MiniOzoneCluster cluster = newCluster(layout)) { + cluster.waitForClusterToBeReady(); + + try (OzoneClient client = cluster.newClient()) { + TestBucket bucket = TestBucket.newBuilder(client).build(); + + testChunkReadBuffers(bucket); + testBufferRelease(bucket); + testCloseReleasesBuffers(bucket); + } + } } @@ -52,12 +70,12 @@ public void testAll() throws Exception { * Test to verify that data read from chunks is stored in a list of buffers * with max capacity equal to the bytes per checksum. */ - public void testChunkReadBuffers() throws Exception { + private void testChunkReadBuffers(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE); - byte[] inputData = writeRandomBytes(keyName, dataLength); + byte[] inputData = bucket.writeRandomBytes(keyName, dataLength); - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { BlockInputStream block0Stream = (BlockInputStream)keyInputStream.getPartStreams().get(0); @@ -74,7 +92,7 @@ public void testChunkReadBuffers() throws Exception { // Read > checksum boundary of data from chunk0 int readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2); byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen); - validateData(inputData, 0, readData); + bucket.validateData(inputData, 0, readData); // The first checksum boundary size of data was already existing in the // ChunkStream buffers. Once that data is read, the next checksum @@ -93,7 +111,7 @@ public void testChunkReadBuffers() throws Exception { readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2); int offset = 2 * BYTES_PER_CHECKSUM + 1; readData = readDataFromChunk(chunk0Stream, offset, readDataLen); - validateData(inputData, offset, readData); + bucket.validateData(inputData, offset, readData); checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 2, 1, BYTES_PER_CHECKSUM); @@ -102,34 +120,34 @@ public void testChunkReadBuffers() throws Exception { // buffers. We read CHUNK_SIZE - 1 as otherwise all the buffers will be // released once all chunk data is read. readData = readDataFromChunk(chunk0Stream, 0, CHUNK_SIZE - 1); - validateData(inputData, 0, readData); + bucket.validateData(inputData, 0, readData); int expectedNumBuffers = CHUNK_SIZE / BYTES_PER_CHECKSUM; checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), expectedNumBuffers, expectedNumBuffers - 1, BYTES_PER_CHECKSUM); // Read the last byte of chunk and verify that the buffers are released. chunk0Stream.read(new byte[1]); - Assert.assertNull("ChunkInputStream did not release buffers after " + - "reaching EOF.", chunk0Stream.getCachedBuffers()); + assertNull(chunk0Stream.getCachedBuffers(), + "ChunkInputStream did not release buffers after reaching EOF."); } } - private void testCloseReleasesBuffers() throws Exception { + private void testCloseReleasesBuffers(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); - writeRandomBytes(keyName, CHUNK_SIZE); + bucket.writeRandomBytes(keyName, CHUNK_SIZE); - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { BlockInputStream block0Stream = (BlockInputStream) keyInputStream.getPartStreams().get(0); block0Stream.initialize(); ChunkInputStream chunk0Stream = block0Stream.getChunkStreams().get(0); readDataFromChunk(chunk0Stream, 0, 1); - Assert.assertNotNull(chunk0Stream.getCachedBuffers()); + assertNotNull(chunk0Stream.getCachedBuffers()); chunk0Stream.close(); - Assert.assertNull(chunk0Stream.getCachedBuffers()); + assertNull(chunk0Stream.getCachedBuffers()); } } @@ -137,12 +155,11 @@ private void testCloseReleasesBuffers() throws Exception { * Test that ChunkInputStream buffers are released as soon as the last byte * of the buffer is read. */ - public void testBufferRelease() throws Exception { + private void testBufferRelease(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); - int dataLength = CHUNK_SIZE; - byte[] inputData = writeRandomBytes(keyName, dataLength); + byte[] inputData = bucket.writeRandomBytes(keyName, CHUNK_SIZE); - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { BlockInputStream block0Stream = (BlockInputStream)keyInputStream.getPartStreams().get(0); @@ -153,21 +170,20 @@ public void testBufferRelease() throws Exception { // Read checksum boundary - 1 bytes of data int readDataLen = BYTES_PER_CHECKSUM - 1; byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen); - validateData(inputData, 0, readData); + bucket.validateData(inputData, 0, readData); // There should be 1 byte of data remaining in the buffer which is not // yet read. Hence, the buffer should not be released. checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM); - Assert.assertEquals(1, chunk0Stream.getCachedBuffers()[0].remaining()); + assertEquals(1, chunk0Stream.getCachedBuffers()[0].remaining()); // Reading the last byte in the buffer should result in all the buffers // being released. readData = readDataFromChunk(chunk0Stream, 1); - validateData(inputData, readDataLen, readData); - Assert - .assertNull("Chunk stream buffers not released after last byte is " + - "read", chunk0Stream.getCachedBuffers()); + bucket.validateData(inputData, readDataLen, readData); + assertNull(chunk0Stream.getCachedBuffers(), + "Chunk stream buffers not released after last byte is read"); // Read more data to get the data till the next checksum boundary. readDataLen = BYTES_PER_CHECKSUM / 2; @@ -177,7 +193,7 @@ public void testBufferRelease() throws Exception { checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM); ByteBuffer lastCachedBuffer = chunk0Stream.getCachedBuffers()[0]; - Assert.assertEquals(BYTES_PER_CHECKSUM - readDataLen, + assertEquals(BYTES_PER_CHECKSUM - readDataLen, lastCachedBuffer.remaining()); // Read more than the remaining data in buffer (but less than the next @@ -185,14 +201,14 @@ public void testBufferRelease() throws Exception { int position = (int) chunk0Stream.getPos(); readDataLen = lastCachedBuffer.remaining() + BYTES_PER_CHECKSUM / 2; readData = readDataFromChunk(chunk0Stream, readDataLen); - validateData(inputData, position, readData); + bucket.validateData(inputData, position, readData); // After reading the remaining data in the buffer, the buffer should be // released and next checksum size of data must be read into the buffers checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM); // Verify that the previously cached buffer is released by comparing it // with the current cached buffer - Assert.assertNotEquals(lastCachedBuffer, + assertNotEquals(lastCachedBuffer, chunk0Stream.getCachedBuffers()[0]); } } @@ -224,16 +240,17 @@ private byte[] readDataFromChunk(ChunkInputStream chunkInputStream, private void checkBufferSizeAndCapacity(ByteBuffer[] buffers, int expectedNumBuffers, int numReleasedBuffers, long expectedBufferCapacity) { - Assert.assertEquals("ChunkInputStream does not have expected number of " + - "ByteBuffers", expectedNumBuffers, buffers.length); + assertEquals(expectedNumBuffers, buffers.length, + "ChunkInputStream does not have expected number of " + + "ByteBuffers"); for (int i = 0; i < buffers.length; i++) { if (i <= numReleasedBuffers - 1) { // This buffer should have been released and hence null - Assert.assertNull("ChunkInputStream Buffer not released after being " + - "read", buffers[i]); + assertNull(buffers[i], + "ChunkInputStream Buffer not released after being read"); } else { - Assert.assertEquals("ChunkInputStream ByteBuffer capacity is wrong", - expectedBufferCapacity, buffers[i].capacity()); + assertEquals(expectedBufferCapacity, buffers[i].capacity(), + "ChunkInputStream ByteBuffer capacity is wrong"); } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 5e21cb8403a..22ad4f036cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -16,88 +16,35 @@ */ package org.apache.hadoop.ozone.client.rpc.read; -import java.io.IOException; import java.time.Duration; -import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; -import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.client.ObjectStore; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.io.KeyInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; -import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; -/** - * Common tests for Ozone's {@code InputStream} implementations. - */ -@RunWith(Parameterized.class) -public abstract class TestInputStreamBase { - - private MiniOzoneCluster cluster; - private OzoneConfiguration conf = new OzoneConfiguration(); - private OzoneClient client; - private ObjectStore objectStore; - - private String volumeName; - private String bucketName; - private String keyString; - - private ContainerLayoutVersion containerLayout; - private static final Random RAND = new Random(); +// TODO remove this class, set config as default in integration tests +abstract class TestInputStreamBase { - protected static final int CHUNK_SIZE = 1024 * 1024; // 1MB - protected static final int FLUSH_SIZE = 2 * CHUNK_SIZE; // 2MB - protected static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE; // 4MB - protected static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; // 8MB - protected static final int BYTES_PER_CHECKSUM = 256 * 1024; // 256KB + static final int CHUNK_SIZE = 1024 * 1024; // 1MB + static final int FLUSH_SIZE = 2 * CHUNK_SIZE; // 2MB + static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE; // 4MB + static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; // 8MB + static final int BYTES_PER_CHECKSUM = 256 * 1024; // 256KB - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - - @Parameterized.Parameters - public static Iterable parameters() { - return ContainerLayoutTestInfo.containerLayoutParameters(); - } - - public TestInputStreamBase(ContainerLayoutVersion layout) { - this.containerLayout = layout; - } + protected static MiniOzoneCluster newCluster( + ContainerLayoutVersion containerLayout) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); - /** - * Create a MiniDFSCluster for testing. - * @throws IOException - */ - @Before - public void init() throws Exception { OzoneClientConfig config = new OzoneClientConfig(); config.setBytesPerChecksum(BYTES_PER_CHECKSUM); conf.setFromObject(config); @@ -116,7 +63,7 @@ public void init() throws Exception { repConf.setInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); - cluster = MiniOzoneCluster.newBuilder(conf) + return MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) .setTotalPipelineNumLimit(5) .setBlockSize(BLOCK_SIZE) @@ -125,99 +72,10 @@ public void init() throws Exception { .setStreamBufferMaxSize(MAX_FLUSH_SIZE) .setStreamBufferSizeUnit(StorageUnit.BYTES) .build(); - cluster.waitForClusterToBeReady(); - //the easiest way to create an open container is creating a key - client = OzoneClientFactory.getRpcClient(conf); - objectStore = client.getObjectStore(); - - volumeName = UUID.randomUUID().toString(); - bucketName = UUID.randomUUID().toString(); - keyString = UUID.randomUUID().toString(); - - objectStore.createVolume(volumeName); - objectStore.getVolume(volumeName).createBucket(bucketName); } - /** - * Shutdown MiniDFSCluster. - */ - @After - public void shutdown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } - - MiniOzoneCluster getCluster() { - return cluster; - } - - String getVolumeName() { - return volumeName; - } - - String getBucketName() { - return bucketName; - } - - KeyInputStream getKeyInputStream(String keyName) throws IOException { - return (KeyInputStream) objectStore - .getVolume(volumeName) - .getBucket(bucketName) - .readKey(keyName).getInputStream(); - } - - String getNewKeyName() { + static String getNewKeyName() { return UUID.randomUUID().toString(); } - byte[] writeKey(String keyName, int dataLength) throws Exception { - ReplicationConfig repConfig = RatisReplicationConfig.getInstance(THREE); - return writeKey(keyName, repConfig, dataLength); - } - - byte[] writeKey(String keyName, ReplicationConfig repConfig, int dataLength) - throws Exception { - OzoneOutputStream key = TestHelper.createKey(keyName, - repConfig, 0, objectStore, volumeName, bucketName); - - byte[] inputData = ContainerTestHelper.getFixedLengthString( - keyString, dataLength).getBytes(UTF_8); - key.write(inputData); - key.close(); - - return inputData; - } - - byte[] writeRandomBytes(String keyName, int dataLength) - throws Exception { - ReplicationConfig repConfig = RatisReplicationConfig.getInstance(THREE); - return writeRandomBytes(keyName, repConfig, dataLength); - } - - byte[] writeRandomBytes(String keyName, ReplicationConfig repConfig, - int dataLength) - throws Exception { - OzoneOutputStream key = TestHelper.createKey(keyName, - repConfig, 0, objectStore, volumeName, bucketName); - - byte[] inputData = new byte[dataLength]; - RAND.nextBytes(inputData); - key.write(inputData); - key.close(); - - return inputData; - } - - void validateData(byte[] inputData, int offset, byte[] readData) { - int readDataLen = readData.length; - byte[] expectedData = new byte[readDataLen]; - System.arraycopy(inputData, (int) offset, expectedData, 0, readDataLen); - - for (int i = 0; i < readDataLen; i++) { - Assert.assertEquals("Read data at does not match the input data at " + - "position " + (offset + i), expectedData[i], readData[i]); - } - } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java index 3899cd6cad7..bfcc0bdd7b0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java @@ -24,76 +24,70 @@ import java.util.List; import java.util.Random; -import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientMetrics; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.storage.BlockExtendedInputStream; import org.apache.hadoop.hdds.scm.storage.BlockInputStream; import org.apache.hadoop.hdds.scm.storage.ChunkInputStream; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.KeyInputStream; import org.apache.hadoop.ozone.common.utils.BufferUtils; import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; +import org.apache.hadoop.ozone.om.TestBucket; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.ozone.container.TestHelper.countReplicas; -import static org.junit.Assert.fail; +import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_BLOCK; +import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_CHUNK; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Tests {@link KeyInputStream}. */ -public class TestKeyInputStream extends TestInputStreamBase { +class TestKeyInputStream extends TestInputStreamBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestKeyInputStream.class); - - public TestKeyInputStream(ContainerLayoutVersion layout) { - super(layout); + private static List layouts() { + return ContainerLayoutVersion.getAllVersions(); } /** * This method does random seeks and reads and validates the reads are * correct or not. - * @param dataLength - * @param keyInputStream - * @param inputData - * @throws Exception */ - private void randomSeek(int dataLength, KeyInputStream keyInputStream, - byte[] inputData) throws Exception { + private void randomSeek(TestBucket bucket, int dataLength, + KeyInputStream keyInputStream, byte[] inputData) throws Exception { // Do random seek. for (int i = 0; i < dataLength - 300; i += 20) { - validate(keyInputStream, inputData, i, 200); + validate(bucket, keyInputStream, inputData, i, 200); } // Seek to end and read in reverse order. And also this is partial chunks // as readLength is 20, chunk length is 100. for (int i = dataLength - 100; i >= 100; i -= 20) { - validate(keyInputStream, inputData, i, 20); + validate(bucket, keyInputStream, inputData, i, 20); } // Start from begin and seek such that we read partially chunks. for (int i = 0; i < dataLength - 300; i += 20) { - validate(keyInputStream, inputData, i, 90); + validate(bucket, keyInputStream, inputData, i, 90); } } @@ -101,87 +95,87 @@ private void randomSeek(int dataLength, KeyInputStream keyInputStream, /** * This method does random seeks and reads and validates the reads are * correct or not. - * @param dataLength - * @param keyInputStream - * @param inputData - * @param readSize - * @throws Exception */ - private void randomPositionSeek(int dataLength, KeyInputStream keyInputStream, + private void randomPositionSeek(TestBucket bucket, int dataLength, + KeyInputStream keyInputStream, byte[] inputData, int readSize) throws Exception { Random rand = new Random(); for (int i = 0; i < 100; i++) { int position = rand.nextInt(dataLength - readSize); - validate(keyInputStream, inputData, position, readSize); + validate(bucket, keyInputStream, inputData, position, readSize); } } /** * This method seeks to specified seek value and read the data specified by * readLength and validate the read is correct or not. - * @param keyInputStream - * @param inputData - * @param seek - * @param readLength - * @throws Exception */ - private void validate(KeyInputStream keyInputStream, byte[] inputData, - long seek, int readLength) throws Exception { + private void validate(TestBucket bucket, KeyInputStream keyInputStream, + byte[] inputData, long seek, int readLength) throws Exception { keyInputStream.seek(seek); byte[] readData = new byte[readLength]; keyInputStream.read(readData, 0, readLength); - validateData(inputData, (int) seek, readData); + bucket.validateData(inputData, (int) seek, readData); } /** * This test runs the others as a single test, so to avoid creating a new * mini-cluster for each test. */ - @Test - public void testNonReplicationReads() throws Exception { - testInputStreams(); - testSeekRandomly(); - testSeek(); - testReadChunkWithByteArray(); - testReadChunkWithByteBuffer(); - testSkip(); - testECSeek(); + @ParameterizedTest + @MethodSource("layouts") + void testNonReplicationReads(ContainerLayoutVersion layout) throws Exception { + try (MiniOzoneCluster cluster = newCluster(layout)) { + cluster.waitForClusterToBeReady(); + + try (OzoneClient client = cluster.newClient()) { + TestBucket bucket = TestBucket.newBuilder(client).build(); + + testInputStreams(bucket); + testSeekRandomly(bucket); + testSeek(bucket); + testReadChunkWithByteArray(bucket); + testReadChunkWithByteBuffer(bucket); + testSkip(bucket); + testECSeek(bucket); + } + } } - public void testInputStreams() throws Exception { + private void testInputStreams(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE) + 1; - writeRandomBytes(keyName, dataLength); + bucket.writeRandomBytes(keyName, dataLength); - KeyInputStream keyInputStream = getKeyInputStream(keyName); + KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName); // Verify BlockStreams and ChunkStreams int expectedNumBlockStreams = BufferUtils.getNumberOfBins( dataLength, BLOCK_SIZE); List blockStreams = keyInputStream.getPartStreams(); - Assert.assertEquals(expectedNumBlockStreams, blockStreams.size()); + assertEquals(expectedNumBlockStreams, blockStreams.size()); int readBlockLength = 0; for (BlockExtendedInputStream stream : blockStreams) { BlockInputStream blockStream = (BlockInputStream) stream; int blockStreamLength = Math.min(BLOCK_SIZE, dataLength - readBlockLength); - Assert.assertEquals(blockStreamLength, blockStream.getLength()); + assertEquals(blockStreamLength, blockStream.getLength()); int expectedNumChunkStreams = BufferUtils.getNumberOfBins(blockStreamLength, CHUNK_SIZE); blockStream.initialize(); List chunkStreams = blockStream.getChunkStreams(); - Assert.assertEquals(expectedNumChunkStreams, chunkStreams.size()); + assertEquals(expectedNumChunkStreams, chunkStreams.size()); int readChunkLength = 0; for (ChunkInputStream chunkStream : chunkStreams) { int chunkStreamLength = Math.min(CHUNK_SIZE, blockStreamLength - readChunkLength); - Assert.assertEquals(chunkStreamLength, chunkStream.getRemaining()); + assertEquals(chunkStreamLength, chunkStream.getRemaining()); readChunkLength += chunkStreamLength; } @@ -190,37 +184,37 @@ public void testInputStreams() throws Exception { } } - public void testSeekRandomly() throws Exception { + private void testSeekRandomly(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE); - byte[] inputData = writeRandomBytes(keyName, dataLength); + byte[] inputData = bucket.writeRandomBytes(keyName, dataLength); - KeyInputStream keyInputStream = getKeyInputStream(keyName); + KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName); // Seek to some where end. - validate(keyInputStream, inputData, dataLength - 200, 100); + validate(bucket, keyInputStream, inputData, dataLength - 200, 100); // Now seek to start. - validate(keyInputStream, inputData, 0, 140); + validate(bucket, keyInputStream, inputData, 0, 140); - validate(keyInputStream, inputData, 200, 300); + validate(bucket, keyInputStream, inputData, 200, 300); - validate(keyInputStream, inputData, 30, 500); + validate(bucket, keyInputStream, inputData, 30, 500); - randomSeek(dataLength, keyInputStream, inputData); + randomSeek(bucket, dataLength, keyInputStream, inputData); // Read entire key. - validate(keyInputStream, inputData, 0, dataLength); + validate(bucket, keyInputStream, inputData, 0, dataLength); // Repeat again and check. - randomSeek(dataLength, keyInputStream, inputData); + randomSeek(bucket, dataLength, keyInputStream, inputData); - validate(keyInputStream, inputData, 0, dataLength); + validate(bucket, keyInputStream, inputData, 0, dataLength); keyInputStream.close(); } - public void testECSeek() throws Exception { + public void testECSeek(TestBucket bucket) throws Exception { int ecChunkSize = 1024 * 1024; ECReplicationConfig repConfig = new ECReplicationConfig(3, 2, RS, ecChunkSize); @@ -228,29 +222,30 @@ public void testECSeek() throws Exception { // 3 full EC blocks plus one chunk int dataLength = (9 * BLOCK_SIZE + ecChunkSize); - byte[] inputData = writeRandomBytes(keyName, repConfig, dataLength); - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + byte[] inputData = bucket.writeRandomBytes(keyName, repConfig, dataLength); + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { - validate(keyInputStream, inputData, 0, ecChunkSize + 1234); + validate(bucket, keyInputStream, inputData, 0, ecChunkSize + 1234); - validate(keyInputStream, inputData, 200, ecChunkSize); + validate(bucket, keyInputStream, inputData, 200, ecChunkSize); - validate(keyInputStream, inputData, BLOCK_SIZE * 4, ecChunkSize); + validate(bucket, keyInputStream, inputData, BLOCK_SIZE * 4, ecChunkSize); - validate(keyInputStream, inputData, BLOCK_SIZE * 4 + 200, ecChunkSize); + validate(bucket, keyInputStream, inputData, + BLOCK_SIZE * 4 + 200, ecChunkSize); - validate(keyInputStream, inputData, dataLength - ecChunkSize - 100, - ecChunkSize + 50); + validate(bucket, keyInputStream, inputData, + dataLength - ecChunkSize - 100, ecChunkSize + 50); - randomPositionSeek(dataLength, keyInputStream, inputData, + randomPositionSeek(bucket, dataLength, keyInputStream, inputData, ecChunkSize + 200); // Read entire key. - validate(keyInputStream, inputData, 0, dataLength); + validate(bucket, keyInputStream, inputData, 0, dataLength); } } - public void testSeek() throws Exception { + public void testSeek(TestBucket bucket) throws Exception { XceiverClientManager.resetXceiverClientMetrics(); XceiverClientMetrics metrics = XceiverClientManager .getXceiverClientMetrics(); @@ -262,28 +257,28 @@ public void testSeek() throws Exception { String keyName = getNewKeyName(); // write data spanning 3 chunks int dataLength = (2 * CHUNK_SIZE) + (CHUNK_SIZE / 2); - byte[] inputData = writeKey(keyName, dataLength); + byte[] inputData = bucket.writeKey(keyName, dataLength); - Assert.assertEquals(writeChunkCount + 3, + assertEquals(writeChunkCount + 3, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk)); - KeyInputStream keyInputStream = getKeyInputStream(keyName); + KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName); // Seek to position 150 keyInputStream.seek(150); - Assert.assertEquals(150, keyInputStream.getPos()); + assertEquals(150, keyInputStream.getPos()); // Seek operation should not result in any readChunk operation. - Assert.assertEquals(readChunkCount, metrics + assertEquals(readChunkCount, metrics .getContainerOpCountMetrics(ContainerProtos.Type.ReadChunk)); byte[] readData = new byte[CHUNK_SIZE]; keyInputStream.read(readData, 0, CHUNK_SIZE); - // Since we reading data from index 150 to 250 and the chunk boundary is + // Since we read data from index 150 to 250 and the chunk boundary is // 100 bytes, we need to read 2 chunks. - Assert.assertEquals(readChunkCount + 2, + assertEquals(readChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.ReadChunk)); keyInputStream.close(); @@ -291,19 +286,19 @@ public void testSeek() throws Exception { // Verify that the data read matches with the input data at corresponding // indices. for (int i = 0; i < CHUNK_SIZE; i++) { - Assert.assertEquals(inputData[CHUNK_SIZE + 50 + i], readData[i]); + assertEquals(inputData[CHUNK_SIZE + 50 + i], readData[i]); } } - public void testReadChunkWithByteArray() throws Exception { + private void testReadChunkWithByteArray(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); // write data spanning multiple blocks/chunks int dataLength = 2 * BLOCK_SIZE + (BLOCK_SIZE / 2); - byte[] data = writeRandomBytes(keyName, dataLength); + byte[] data = bucket.writeRandomBytes(keyName, dataLength); // read chunk data using Byte Array - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { int[] bufferSizeList = {BYTES_PER_CHECKSUM + 1, CHUNK_SIZE / 4, CHUNK_SIZE / 2, CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, @@ -315,15 +310,15 @@ public void testReadChunkWithByteArray() throws Exception { } } - public void testReadChunkWithByteBuffer() throws Exception { + public void testReadChunkWithByteBuffer(TestBucket bucket) throws Exception { String keyName = getNewKeyName(); // write data spanning multiple blocks/chunks int dataLength = 2 * BLOCK_SIZE + (BLOCK_SIZE / 2); - byte[] data = writeRandomBytes(keyName, dataLength); + byte[] data = bucket.writeRandomBytes(keyName, dataLength); // read chunk data using ByteBuffer - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { int[] bufferSizeList = {BYTES_PER_CHECKSUM + 1, CHUNK_SIZE / 4, CHUNK_SIZE / 2, CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, @@ -335,7 +330,7 @@ public void testReadChunkWithByteBuffer() throws Exception { } } - public void testSkip() throws Exception { + private void testSkip(TestBucket bucket) throws Exception { XceiverClientManager.resetXceiverClientMetrics(); XceiverClientMetrics metrics = XceiverClientManager .getXceiverClientMetrics(); @@ -347,28 +342,28 @@ public void testSkip() throws Exception { String keyName = getNewKeyName(); // write data spanning 3 chunks int dataLength = (2 * CHUNK_SIZE) + (CHUNK_SIZE / 2); - byte[] inputData = writeKey(keyName, dataLength); + byte[] inputData = bucket.writeKey(keyName, dataLength); - Assert.assertEquals(writeChunkCount + 3, + assertEquals(writeChunkCount + 3, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk)); - KeyInputStream keyInputStream = getKeyInputStream(keyName); + KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName); // skip 150 long skipped = keyInputStream.skip(70); - Assert.assertEquals(70, skipped); - Assert.assertEquals(70, keyInputStream.getPos()); + assertEquals(70, skipped); + assertEquals(70, keyInputStream.getPos()); skipped = keyInputStream.skip(0); - Assert.assertEquals(0, skipped); - Assert.assertEquals(70, keyInputStream.getPos()); + assertEquals(0, skipped); + assertEquals(70, keyInputStream.getPos()); skipped = keyInputStream.skip(80); - Assert.assertEquals(80, skipped); - Assert.assertEquals(150, keyInputStream.getPos()); + assertEquals(80, skipped); + assertEquals(150, keyInputStream.getPos()); // Skip operation should not result in any readChunk operation. - Assert.assertEquals(readChunkCount, metrics + assertEquals(readChunkCount, metrics .getContainerOpCountMetrics(ContainerProtos.Type.ReadChunk)); byte[] readData = new byte[CHUNK_SIZE]; @@ -376,7 +371,7 @@ public void testSkip() throws Exception { // Since we reading data from index 150 to 250 and the chunk boundary is // 100 bytes, we need to read 2 chunks. - Assert.assertEquals(readChunkCount + 2, + assertEquals(readChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.ReadChunk)); keyInputStream.close(); @@ -384,94 +379,87 @@ public void testSkip() throws Exception { // Verify that the data read matches with the input data at corresponding // indices. for (int i = 0; i < CHUNK_SIZE; i++) { - Assert.assertEquals(inputData[CHUNK_SIZE + 50 + i], readData[i]); + assertEquals(inputData[CHUNK_SIZE + 50 + i], readData[i]); } } - @Test - public void readAfterReplication() throws Exception { - testReadAfterReplication(false); + private static List readAfterReplicationArgs() { + return Arrays.asList( + Arguments.arguments(FILE_PER_BLOCK, false), + Arguments.arguments(FILE_PER_BLOCK, true), + Arguments.arguments(FILE_PER_CHUNK, false), + Arguments.arguments(FILE_PER_CHUNK, true) + ); } - @Test - public void readAfterReplicationWithUnbuffering() throws Exception { - testReadAfterReplication(true); - } + @ParameterizedTest + @MethodSource("readAfterReplicationArgs") + void readAfterReplication(ContainerLayoutVersion layout, + boolean doUnbuffer) throws Exception { + try (MiniOzoneCluster cluster = newCluster(layout)) { + cluster.waitForClusterToBeReady(); + + try (OzoneClient client = cluster.newClient()) { + TestBucket bucket = TestBucket.newBuilder(client).build(); - private void testReadAfterReplication(boolean doUnbuffer) throws Exception { - Assume.assumeTrue(getCluster().getHddsDatanodes().size() > 3); + testReadAfterReplication(cluster, bucket, doUnbuffer); + } + } + } + private void testReadAfterReplication(MiniOzoneCluster cluster, + TestBucket bucket, boolean doUnbuffer) throws Exception { int dataLength = 2 * CHUNK_SIZE; String keyName = getNewKeyName(); - byte[] data = writeRandomBytes(keyName, dataLength); + byte[] data = bucket.writeRandomBytes(keyName, dataLength); - OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(getVolumeName()) - .setBucketName(getBucketName()) + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(bucket.delegate().getVolumeName()) + .setBucketName(bucket.delegate().getName()) .setKeyName(keyName) .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) .build(); - OmKeyInfo keyInfo = getCluster().getOzoneManager() + OmKeyInfo keyInfo = cluster.getOzoneManager() .getKeyInfo(keyArgs, false) .getKeyInfo(); OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); - Assert.assertNotNull(locations); + assertNotNull(locations); List locationInfoList = locations.getLocationList(); - Assert.assertEquals(1, locationInfoList.size()); + assertEquals(1, locationInfoList.size()); OmKeyLocationInfo loc = locationInfoList.get(0); long containerID = loc.getContainerID(); - Assert.assertEquals(3, countReplicas(containerID, getCluster())); + assertEquals(3, countReplicas(containerID, cluster)); - TestHelper.waitForContainerClose(getCluster(), containerID); + TestHelper.waitForContainerClose(cluster, containerID); List pipelineNodes = loc.getPipeline().getNodes(); // read chunk data - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { int b = keyInputStream.read(); - Assert.assertNotEquals(-1, b); + assertNotEquals(-1, b); if (doUnbuffer) { keyInputStream.unbuffer(); } - getCluster().shutdownHddsDatanode(pipelineNodes.get(0)); + cluster.shutdownHddsDatanode(pipelineNodes.get(0)); // check that we can still read it assertReadFully(data, keyInputStream, dataLength - 1, 1); } // read chunk data with ByteBuffer - try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) { + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { int b = keyInputStream.read(); - Assert.assertNotEquals(-1, b); + assertNotEquals(-1, b); if (doUnbuffer) { keyInputStream.unbuffer(); } - getCluster().shutdownHddsDatanode(pipelineNodes.get(0)); + cluster.shutdownHddsDatanode(pipelineNodes.get(0)); // check that we can still read it assertReadFullyUsingByteBuffer(data, keyInputStream, dataLength - 1, 1); } } - private void waitForNodeToBecomeDead( - DatanodeDetails datanode) throws TimeoutException, InterruptedException { - GenericTestUtils.waitFor(() -> - HddsProtos.NodeState.DEAD == getNodeHealth(datanode), - 100, 30000); - LOG.info("Node {} is {}", datanode.getUuidString(), - getNodeHealth(datanode)); - } - - private HddsProtos.NodeState getNodeHealth(DatanodeDetails dn) { - HddsProtos.NodeState health = null; - try { - NodeManager nodeManager = - getCluster().getStorageContainerManager().getScmNodeManager(); - health = nodeManager.getNodeStatus(dn).getHealth(); - } catch (NodeNotFoundException e) { - fail("Unexpected NodeNotFound exception"); - } - return health; - } - private void assertReadFully(byte[] data, InputStream in, int bufferSize, int totalRead) throws IOException { @@ -485,10 +473,10 @@ private void assertReadFully(byte[] data, InputStream in, Arrays.copyOfRange(data, totalRead, totalRead + numBytesRead); byte[] tmp2 = Arrays.copyOfRange(buffer, 0, numBytesRead); - Assert.assertArrayEquals(tmp1, tmp2); + assertArrayEquals(tmp1, tmp2); totalRead += numBytesRead; } - Assert.assertEquals(data.length, totalRead); + assertEquals(data.length, totalRead); } private void assertReadFullyUsingByteBuffer(byte[] data, KeyInputStream in, @@ -505,10 +493,10 @@ private void assertReadFullyUsingByteBuffer(byte[] data, KeyInputStream in, byte[] tmp2 = new byte[numBytesRead]; buffer.flip(); buffer.get(tmp2, 0, numBytesRead); - Assert.assertArrayEquals(tmp1, tmp2); + assertArrayEquals(tmp1, tmp2); totalRead += numBytesRead; buffer.clear(); } - Assert.assertEquals(data.length, totalRead); + assertEquals(data.length, totalRead); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java index c7460623044..28d19d0be87 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java @@ -19,177 +19,142 @@ package org.apache.hadoop.ozone.container; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerClose; import static org.apache.hadoop.ozone.container.TestHelper.waitForReplicaCount; -import static org.junit.Assert.assertFalse; +import static org.apache.ozone.test.GenericTestUtils.setLogLevel; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; import java.io.OutputStream; import java.time.Duration; -import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom; -import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; /** * Tests ozone containers replication. */ -@RunWith(Parameterized.class) -public class TestContainerReplication { - /** - * Set the timeout for every test. - */ - @Rule - public TestRule testTimeout = new JUnit5AwareTimeout(Timeout.seconds(300)); +@Timeout(300) +class TestContainerReplication { private static final String VOLUME = "vol1"; private static final String BUCKET = "bucket1"; private static final String KEY = "key1"; - private MiniOzoneCluster cluster; - private OzoneClient client; - private String placementPolicyClass; - - @Parameterized.Parameters - public static List parameters() { - List classes = new ArrayList<>(); - classes.add(SCMContainerPlacementRackAware.class.getCanonicalName()); - classes.add(SCMContainerPlacementCapacity.class.getCanonicalName()); - classes.add(SCMContainerPlacementRandom.class.getCanonicalName()); - return classes; - } - - public TestContainerReplication(String placementPolicy) { - this.placementPolicyClass = placementPolicy; + private static final List> POLICIES = asList( + SCMContainerPlacementCapacity.class, + SCMContainerPlacementRackAware.class, + SCMContainerPlacementRandom.class + ); + + static List containerReplicationArguments() { + List arguments = new LinkedList<>(); + for (Class policyClass : POLICIES) { + String canonicalName = policyClass.getCanonicalName(); + arguments.add(Arguments.arguments(canonicalName, true)); + arguments.add(Arguments.arguments(canonicalName, false)); + } + return arguments; } - @Before - public void setUp() throws Exception { - GenericTestUtils.setLogLevel(SCMContainerPlacementRandom.LOG, Level.DEBUG); - GenericTestUtils.setLogLevel(SCMContainerPlacementCapacity.LOG, - Level.DEBUG); - GenericTestUtils.setLogLevel(SCMContainerPlacementRackAware.LOG, - Level.DEBUG); + @BeforeAll + static void setUp() { + setLogLevel(SCMContainerPlacementCapacity.LOG, Level.DEBUG); + setLogLevel(SCMContainerPlacementRackAware.LOG, Level.DEBUG); + setLogLevel(SCMContainerPlacementRandom.LOG, Level.DEBUG); } - @After - public void tearDown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } + @ParameterizedTest + @MethodSource("containerReplicationArguments") + void testContainerReplication( + String placementPolicyClass, boolean legacyEnabled) throws Exception { - @Test - public void testContainerReplication() throws Exception { - OzoneConfiguration conf = createConfiguration(); + OzoneConfiguration conf = createConfiguration(legacyEnabled); conf.set(OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, placementPolicyClass); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); - cluster.waitForClusterToBeReady(); - client = OzoneClientFactory.getRpcClient(conf); - createTestData(); + try (MiniOzoneCluster cluster = newCluster(conf)) { + cluster.waitForClusterToBeReady(); + try (OzoneClient client = cluster.newClient()) { + createTestData(client); - List keyLocations = lookupKey(cluster); - assertFalse(keyLocations.isEmpty()); + List keyLocations = lookupKey(cluster); + assertFalse(keyLocations.isEmpty()); - OmKeyLocationInfo keyLocation = keyLocations.get(0); - long containerID = keyLocation.getContainerID(); - waitForContainerClose(cluster, containerID); + OmKeyLocationInfo keyLocation = keyLocations.get(0); + long containerID = keyLocation.getContainerID(); + waitForContainerClose(cluster, containerID); - cluster.shutdownHddsDatanode(keyLocation.getPipeline().getFirstNode()); - waitForReplicaCount(containerID, 2, cluster); + cluster.shutdownHddsDatanode(keyLocation.getPipeline().getFirstNode()); + waitForReplicaCount(containerID, 2, cluster); - waitForReplicaCount(containerID, 3, cluster); + waitForReplicaCount(containerID, 3, cluster); + } + } } - @Test - public void testContainerReplicationWithLegacyReplicationManagerDisabled() - throws Exception { - OzoneConfiguration conf = createConfiguration(); - - /* - Disable LegacyReplicationManager so that ReplicationManager handles Ratis - containers. - */ - ReplicationManagerConfiguration repConf = - conf.getObject(ReplicationManagerConfiguration.class); - repConf.setEnableLegacy(false); - repConf.setUnderReplicatedInterval(Duration.ofSeconds(1)); - conf.setFromObject(repConf); - - conf.set(OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, placementPolicyClass); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); - cluster.waitForClusterToBeReady(); - client = OzoneClientFactory.getRpcClient(conf); - createTestData(); - - List keyLocations = lookupKey(cluster); - assertFalse(keyLocations.isEmpty()); - - OmKeyLocationInfo keyLocation = keyLocations.get(0); - long containerID = keyLocation.getContainerID(); - waitForContainerClose(cluster, containerID); - - cluster.shutdownHddsDatanode(keyLocation.getPipeline().getFirstNode()); - waitForReplicaCount(containerID, 2, cluster); - - waitForReplicaCount(containerID, 3, cluster); + private static MiniOzoneCluster newCluster(OzoneConfiguration conf) + throws IOException { + return MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .build(); } - private static OzoneConfiguration createConfiguration() { + private static OzoneConfiguration createConfiguration(boolean enableLegacy) { OzoneConfiguration conf = new OzoneConfiguration(); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, TimeUnit.SECONDS); ReplicationManagerConfiguration repConf = conf.getObject(ReplicationManagerConfiguration.class); + repConf.setEnableLegacy(enableLegacy); repConf.setInterval(Duration.ofSeconds(1)); + repConf.setUnderReplicatedInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); + return conf; } - private void createTestData() throws IOException { + // TODO use common helper to create test data + private void createTestData(OzoneClient client) throws IOException { ObjectStore objectStore = client.getObjectStore(); objectStore.createVolume(VOLUME); OzoneVolume volume = objectStore.getVolume(VOLUME); volume.createBucket(BUCKET); OzoneBucket bucket = volume.getBucket(BUCKET); - try (OutputStream out = bucket.createKey(KEY, 0)) { + + try (OutputStream out = bucket.createKey(KEY, 0, + RatisReplicationConfig.getInstance(THREE), emptyMap())) { out.write("Hello".getBytes(UTF_8)); } } @@ -200,11 +165,10 @@ private static List lookupKey(MiniOzoneCluster cluster) .setVolumeName(VOLUME) .setBucketName(BUCKET) .setKeyName(KEY) - .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) .build(); OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs); OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); - Assert.assertNotNull(locations); + assertNotNull(locations); return locations.getLocationList(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index 828f6c2964b..06426ca922f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.container.ozoneimpl; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; @@ -38,41 +37,29 @@ import org.apache.hadoop.hdds.scm.XceiverClientGrpc; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.ozone.container.common.ContainerTestUtils; +import org.apache.hadoop.ozone.container.replication.ContainerDownloader; import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Timeout; + +import java.io.IOException; import java.nio.file.Path; import java.security.cert.CertificateExpiredException; import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Date; -import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT; +import static java.util.Collections.singletonList; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ACK_TIMEOUT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL; @@ -81,58 +68,44 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_CONTAINER_DB_DIR; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; +import static org.apache.hadoop.ozone.container.ContainerTestHelper.getTestContainerID; +import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getMockContext; import static org.apache.hadoop.ozone.container.replication.CopyContainerCompression.NO_COMPRESSION; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests ozone containers via secure grpc/netty. */ -@RunWith(Parameterized.class) -public class TestOzoneContainerWithTLS { - private static final Logger LOG = LoggerFactory.getLogger( - TestOzoneContainerWithTLS.class); - /** - * Set the timeout for every test. - */ - @Rule - public TestRule testTimeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); +@Timeout(300) +class TestOzoneContainerWithTLS { + + @TempDir + private Path tempFolder; private OzoneConfiguration conf; private ContainerTokenSecretManager secretManager; private CertificateClientTestImpl caClient; private SecretKeyClient secretKeyClient; - private boolean containerTokenEnabled; - private int certLifetime = 15 * 1000; // 15s - - public TestOzoneContainerWithTLS(boolean enableToken) { - this.containerTokenEnabled = enableToken; - } + private final Duration certLifetime = Duration.ofSeconds(15); - @Parameterized.Parameters - public static Collection enableBlockToken() { - return Arrays.asList(new Object[][] { - {false}, - {true} - }); - } - - @Before - public void setup() throws Exception { + @BeforeEach + void setup() throws Exception { conf = new OzoneConfiguration(); - String ozoneMetaPath = - GenericTestUtils.getTempPath("ozoneMeta"); - File ozoneMetaFile = new File(ozoneMetaPath); - conf.set(OZONE_METADATA_DIRS, ozoneMetaPath); - FileUtil.fullyDelete(ozoneMetaFile); - String keyDirName = conf.get(HDDS_KEY_DIR_NAME, - HDDS_KEY_DIR_NAME_DEFAULT); + conf.set(OZONE_METADATA_DIRS, + tempFolder.resolve("meta").toString()); + conf.set(HDDS_DATANODE_DIR_KEY, + tempFolder.resolve("data").toString()); + conf.set(HDDS_DATANODE_CONTAINER_DB_DIR, + tempFolder.resolve("containers").toString()); - File ozoneKeyDir = new File(ozoneMetaFile, keyDirName); - ozoneKeyDir.mkdirs(); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_ENABLED, true); @@ -140,9 +113,7 @@ public void setup() throws Exception { conf.setBoolean(HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED, false); conf.setInt(HDDS_KEY_LEN, 1024); - // certificate lives for 10s - conf.set(HDDS_X509_DEFAULT_DURATION, - Duration.ofMillis(certLifetime).toString()); + conf.set(HDDS_X509_DEFAULT_DURATION, certLifetime.toString()); conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT2S"); conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT1S"); // 1s conf.set(HDDS_X509_CA_ROTATION_ACK_TIMEOUT, "PT1S"); // 1s @@ -157,42 +128,40 @@ public void setup() throws Exception { secretKeyClient); } - @Test(expected = CertificateExpiredException.class) - public void testCertificateLifetime() throws Exception { - // Sleep to wait for certificate expire - LocalDateTime now = LocalDateTime.now(); - now = now.plusSeconds(certLifetime / 1000); - caClient.getCertificate().checkValidity(Date.from( - now.atZone(ZoneId.systemDefault()).toInstant())); + @Test + void testCertificateLifetime() { + Date afterExpiry = Date.from(LocalDateTime.now() + .plus(certLifetime) + .atZone(ZoneId.systemDefault()) + .toInstant()); + + assertThrows(CertificateExpiredException.class, + () -> caClient.getCertificate().checkValidity(afterExpiry)); } - @Test - public void testCreateOzoneContainer() throws Exception { - LOG.info("testCreateOzoneContainer with TLS and containerToken enabled: {}", - containerTokenEnabled); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void createContainer(boolean containerTokenEnabled) throws Exception { conf.setBoolean(HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED, containerTokenEnabled); - long containerId = ContainerTestHelper.getTestContainerID(); - OzoneContainer container = null; - System.out.println(System.getProperties().getProperty("java.library.path")); + final long containerId = getTestContainerID(); DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + OzoneContainer container = null; try { Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath()); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE) .getValue()); - conf.setBoolean( - OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); - container = new OzoneContainer(dn, conf, ContainerTestUtils - .getMockContext(dn, conf), caClient, secretKeyClient); + container = new OzoneContainer(dn, conf, getMockContext(dn, conf), + caClient, secretKeyClient); //Set scmId and manually start ozone container. container.start(UUID.randomUUID().toString()); try (XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf, - Collections.singletonList(caClient.getCACertificate()))) { + singletonList(caClient.getCACertificate()))) { if (containerTokenEnabled) { client.connect(); @@ -212,13 +181,13 @@ public void testCreateOzoneContainer() throws Exception { } } - @Test - public void testContainerDownload() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void downloadContainer(boolean tokenEnabled) throws Exception { DatanodeDetails dn = MockDatanodeDetails.createDatanodeDetails( UUID.randomUUID().toString(), "localhost", "0.0.0.0", "/default-rack"); Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.newFolder().getPath()); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE) .getValue()); @@ -226,107 +195,78 @@ public void testContainerDownload() throws Exception { OzoneContainer container = null; try { - container = new OzoneContainer(dn, conf, ContainerTestUtils - .getMockContext(dn, conf), caClient, secretKeyClient); + container = new OzoneContainer(dn, conf, + getMockContext(dn, conf), caClient, secretKeyClient); // Set scmId and manually start ozone container. container.start(UUID.randomUUID().toString()); // Create containers - long containerId = ContainerTestHelper.getTestContainerID(); - List containerIdList = new ArrayList<>(); - XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf, - Collections.singletonList(caClient.getCACertificate())); - client.connect(); - if (containerTokenEnabled) { - Token token = secretManager.generateToken( - UserGroupInformation.getCurrentUser().getUserName(), - ContainerID.valueOf(containerId)); - createSecureContainer(client, containerId, token); - closeSecureContainer(client, containerId, token); - } else { - createContainer(client, containerId); - closeContainer(client, containerId); - } - containerIdList.add(containerId++); + try (XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf, + singletonList(caClient.getCACertificate()))) { + client.connect(); + + final long containerId = getTestContainerID(); + createAndCloseContainer(tokenEnabled, containerId, client); - // Wait certificate to expire - GenericTestUtils.waitFor(() -> - caClient.getCertificate().getNotAfter().before(new Date()), - 100, certLifetime); + // Wait certificate to expire + GenericTestUtils.waitFor( + () -> caClient.getCertificate().getNotAfter().before(new Date()), + 100, (int) certLifetime.toMillis()); - List sourceDatanodes = new ArrayList<>(); - sourceDatanodes.add(dn); - if (containerTokenEnabled) { // old client still function well after certificate expired - Token token = secretManager.generateToken( - UserGroupInformation.getCurrentUser().getUserName(), - ContainerID.valueOf(containerId)); - createSecureContainer(client, containerId, token); - closeSecureContainer(client, containerId++, token); - } else { - createContainer(client, containerId); - closeContainer(client, containerId++); - } + createAndCloseContainer(tokenEnabled, getTestContainerID(), client); - // Download newly created container will fail because of cert expired - GenericTestUtils.LogCapturer logCapture = GenericTestUtils.LogCapturer - .captureLogs(SimpleContainerDownloader.LOG); - SimpleContainerDownloader downloader = - new SimpleContainerDownloader(conf, caClient); - Path file = downloader.getContainerDataFromReplicas( - containerId, sourceDatanodes, null, NO_COMPRESSION); - downloader.close(); - Assert.assertNull(file); - Assert.assertTrue(logCapture.getOutput().contains( - "java.security.cert.CertificateExpiredException")); - - // Renew the certificate - caClient.renewKey(); - - // old client still function well after certificate renewed - if (containerTokenEnabled) { - Token token = secretManager.generateToken( - UserGroupInformation.getCurrentUser().getUserName(), - ContainerID.valueOf(containerId)); - createSecureContainer(client, containerId, token); - closeSecureContainer(client, containerId++, token); - } + // Download newly created container will fail because of cert expired + GenericTestUtils.LogCapturer logCapture = GenericTestUtils.LogCapturer + .captureLogs(SimpleContainerDownloader.LOG); + assertNull(downloadContainer(containerId, dn)); + assertThat(logCapture.getOutput(), + containsString(CertificateExpiredException.class.getName())); - // Wait keyManager and trustManager to reload - Thread.sleep(2000); - - // old client still function well after certificate reload - if (containerTokenEnabled) { - Token token = secretManager.generateToken( - UserGroupInformation.getCurrentUser().getUserName(), - ContainerID.valueOf(containerId)); - createSecureContainer(client, containerId, token); - closeSecureContainer(client, containerId++, token); - } else { - createContainer(client, containerId); - closeContainer(client, containerId++); - } + // Renew the certificate + caClient.renewKey(); - // Download container should succeed after key and cert renewed - for (Long cId : containerIdList) { - downloader = new SimpleContainerDownloader(conf, caClient); - try { - file = downloader.getContainerDataFromReplicas(cId, sourceDatanodes, - null, NO_COMPRESSION); - downloader.close(); - Assert.assertNotNull(file); - } finally { - if (downloader != null) { - downloader.close(); - } - client.close(); - } + // old client still function well after certificate renewed + createAndCloseContainer(tokenEnabled, getTestContainerID(), client); + + // Wait keyManager and trustManager to reload + Thread.sleep(2000); // TODO replace + + // old client still function well after certificate reload + createAndCloseContainer(tokenEnabled, getTestContainerID(), client); + + // Download container should succeed after key and cert renewed + assertNotNull(downloadContainer(containerId, dn)); } } finally { if (container != null) { container.stop(); } + // TODO delete leftover hadoop-ozone/integration-test/container.db + } + } + + private Path downloadContainer(long containerId, DatanodeDetails source) + throws IOException { + try (ContainerDownloader downloader = new SimpleContainerDownloader( + conf, caClient)) { + return downloader.getContainerDataFromReplicas(containerId, + singletonList(source), tempFolder.resolve("tmp"), NO_COMPRESSION); + } + } + + private void createAndCloseContainer(boolean containerTokenEnabled, + long containerId, XceiverClientGrpc client) throws Exception { + if (containerTokenEnabled) { + Token token = secretManager.generateToken( + UserGroupInformation.getCurrentUser().getUserName(), + ContainerID.valueOf(containerId)); + createSecureContainer(client, containerId, token); + closeSecureContainer(client, containerId, token); + } else { + createContainer(client, containerId); + closeContainer(client, containerId); } } @@ -335,8 +275,8 @@ public static void createContainer(XceiverClientSpi client, ContainerCommandRequestProto request = ContainerTestHelper .getCreateContainerRequest(containerID, client.getPipeline()); ContainerCommandResponseProto response = client.sendCommand(request); - Assert.assertNotNull(response); - Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS); + assertNotNull(response); + assertSame(ContainerProtos.Result.SUCCESS, response.getResult()); } public static void createSecureContainer(XceiverClientSpi client, @@ -347,8 +287,8 @@ public static void createSecureContainer(XceiverClientSpi client, containerID, client.getPipeline(), token); ContainerCommandResponseProto response = client.sendCommand(request); - Assert.assertNotNull(response); - Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS); + assertNotNull(response); + assertSame(ContainerProtos.Result.SUCCESS, response.getResult()); } public static void closeContainer(XceiverClientSpi client, @@ -356,8 +296,8 @@ public static void closeContainer(XceiverClientSpi client, ContainerCommandRequestProto request = ContainerTestHelper .getCloseContainer(client.getPipeline(), containerID); ContainerCommandResponseProto response = client.sendCommand(request); - Assert.assertNotNull(response); - Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS); + assertNotNull(response); + assertSame(ContainerProtos.Result.SUCCESS, response.getResult()); } public static void closeSecureContainer(XceiverClientSpi client, @@ -368,7 +308,7 @@ public static void closeSecureContainer(XceiverClientSpi client, containerID, token); ContainerCommandResponseProto response = client.sendCommand(request); - Assert.assertNotNull(response); - Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS); + assertNotNull(response); + assertSame(ContainerProtos.Result.SUCCESS, response.getResult()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index b3f87dc7b13..715b0678a17 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier; import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager; -import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -43,19 +42,18 @@ import org.apache.hadoop.security.token.Token; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.Path; import java.security.PrivilegedAction; import java.time.Instant; import java.util.Arrays; @@ -75,74 +73,62 @@ /** * Tests ozone containers via secure grpc/netty. */ -@RunWith(Parameterized.class) -public class TestSecureOzoneContainer { +@Timeout(300) +class TestSecureOzoneContainer { private static final Logger LOG = LoggerFactory.getLogger( TestSecureOzoneContainer.class); - /** - * Set the timeout for every test. - */ - @Rule - public TestRule testTimeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); + @TempDir + private Path tempFolder; private OzoneConfiguration conf; - private SecurityConfig secConfig; - private final boolean requireToken; - private final boolean hasToken; - private final boolean tokenExpired; private CertificateClientTestImpl caClient; private SecretKeyClient secretKeyClient; private ContainerTokenSecretManager secretManager; - public TestSecureOzoneContainer(Boolean requireToken, - Boolean hasToken, Boolean tokenExpired) { - this.requireToken = requireToken; - this.hasToken = hasToken; - this.tokenExpired = tokenExpired; + static Collection blockTokenOptions() { + return Arrays.asList( + Arguments.arguments(true, true, false), + Arguments.arguments(true, true, true), + Arguments.arguments(true, false, false), + Arguments.arguments(false, true, false), + Arguments.arguments(false, false, false) + ); } - @Parameterized.Parameters - public static Collection blockTokenOptions() { - return Arrays.asList(new Object[][] { - {true, true, false}, - {true, true, true}, - {true, false, false}, - {false, true, false}, - {false, false, false}}); - } - - @Before - public void setup() throws Exception { + @BeforeAll + static void init() { DefaultMetricsSystem.setMiniClusterMode(true); ExitUtils.disableSystemExit(); + } + + @BeforeEach + void setup() throws Exception { conf = new OzoneConfiguration(); String ozoneMetaPath = GenericTestUtils.getTempPath("ozoneMeta"); conf.set(OZONE_METADATA_DIRS, ozoneMetaPath); - secConfig = new SecurityConfig(conf); caClient = new CertificateClientTestImpl(conf); secretKeyClient = new SecretKeyTestClient(); secretManager = new ContainerTokenSecretManager( TimeUnit.DAYS.toMillis(1), secretKeyClient); } - @Test - public void testCreateOzoneContainer() throws Exception { - LOG.info("Test case: requireBlockToken: {} hasBlockToken: {} " + - "blockTokenExpired: {}.", requireToken, hasToken, - tokenExpired); + @ParameterizedTest + @MethodSource("blockTokenOptions") + void testCreateOzoneContainer(boolean requireToken, boolean hasToken, + boolean tokenExpired) throws Exception { + final String testCase = testCase(requireToken, hasToken, tokenExpired); + LOG.info("Test case: {}", testCase); + conf.setBoolean(HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED, requireToken); conf.setBoolean(HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED, requireToken); ContainerID containerID = ContainerID.valueOf(getTestContainerID()); OzoneContainer container = null; - System.out.println(System.getProperties().getProperty("java.library.path")); try { Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath()); + conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.toString()); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE) .getValue()); @@ -183,11 +169,11 @@ public void testCreateOzoneContainer() throws Exception { !requireToken || (hasToken && !tokenExpired) ? ContainerProtos.Result.SUCCESS : ContainerProtos.Result.BLOCK_TOKEN_VERIFICATION_FAILED; - assertEquals(expectedResult, response.getResult(), this::testCase); + assertEquals(expectedResult, response.getResult(), testCase); } catch (SCMSecurityException e) { - assertState(requireToken && hasToken && tokenExpired); + assertTrue(requireToken && hasToken && tokenExpired, testCase); } catch (IOException e) { - assertState(requireToken && !hasToken); + assertTrue(requireToken && !hasToken, testCase); } catch (Exception e) { fail(e); } @@ -200,11 +186,8 @@ public void testCreateOzoneContainer() throws Exception { } } - private void assertState(boolean condition) { - assertTrue(condition, this::testCase); - } - - private String testCase() { + private String testCase(boolean requireToken, boolean hasToken, + boolean tokenExpired) { if (!requireToken) { return "unsecure"; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java index 79a60d7968e..218c35c7d3f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java @@ -20,40 +20,33 @@ package org.apache.hadoop.ozone.dn.scanner; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; +import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.utils.ContainerLogger; import org.apache.hadoop.ozone.container.ozoneimpl.BackgroundContainerDataScanner; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; -import java.util.Collection; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertEquals; + /** * Integration tests for the background container data scanner. This scanner * checks all data and metadata in the container. */ -@RunWith(Parameterized.class) -public class TestBackgroundContainerDataScannerIntegration +class TestBackgroundContainerDataScannerIntegration extends TestContainerScannerIntegrationAbstract { - private final ContainerCorruptions corruption; - private final LogCapturer logCapturer; - - @Parameterized.Parameters(name = "{0}") - public static Collection supportedCorruptionTypes() { - // Background container data scanner should be able to detect all errors. - return ContainerCorruptions.getAllParamsExcept(); - } + private final LogCapturer logCapturer = + LogCapturer.log4j2(ContainerLogger.LOG_NAME); - @BeforeClass - public static void init() throws Exception { + @BeforeAll + static void init() throws Exception { OzoneConfiguration ozoneConfig = new OzoneConfiguration(); ozoneConfig.setBoolean( ContainerScannerConfiguration.HDDS_CONTAINER_SCRUB_ENABLED, true); @@ -69,28 +62,25 @@ public static void init() throws Exception { buildCluster(ozoneConfig); } - public TestBackgroundContainerDataScannerIntegration( - ContainerCorruptions corruption) { - this.corruption = corruption; - logCapturer = GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME); - } - /** * {@link BackgroundContainerDataScanner} should detect corrupted blocks * in a closed container without client interaction. */ - @Test - public void testCorruptionDetected() throws Exception { + @ParameterizedTest + // Background container data scanner should be able to detect all errors. + @EnumSource + void testCorruptionDetected(ContainerCorruptions corruption) + throws Exception { long containerID = writeDataThenCloseContainer(); // Container corruption has not yet been introduced. - Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, - getDnContainer(containerID).getContainerState()); + Container container = getDnContainer(containerID); + assertEquals(State.CLOSED, container.getContainerState()); + + corruption.applyTo(container); - corruption.applyTo(getDnContainer(containerID)); // Wait for the scanner to detect corruption. - GenericTestUtils.waitFor(() -> - getDnContainer(containerID).getContainerState() == - ContainerProtos.ContainerDataProto.State.UNHEALTHY, + GenericTestUtils.waitFor( + () -> container.getContainerState() == State.UNHEALTHY, 500, 5000); // Wait for SCM to get a report of the unhealthy replica. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java index ee0e852f2d9..52da1035cae 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java @@ -20,44 +20,43 @@ package org.apache.hadoop.ozone.dn.scanner; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.utils.ContainerLogger; import org.apache.hadoop.ozone.container.ozoneimpl.BackgroundContainerMetadataScanner; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration; import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.time.Duration; import java.util.Collection; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertEquals; + /** * Integration tests for the background container metadata scanner. This * scanner does a quick check of container metadata to find obvious failures * faster than a full data scan. */ -@RunWith(Parameterized.class) -public class TestBackgroundContainerMetadataScannerIntegration +class TestBackgroundContainerMetadataScannerIntegration extends TestContainerScannerIntegrationAbstract { - private final ContainerCorruptions corruption; - private final GenericTestUtils.LogCapturer logCapturer; + private final GenericTestUtils.LogCapturer logCapturer = + GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME); - @Parameterized.Parameters(name = "{0}") - public static Collection supportedCorruptionTypes() { + static Collection supportedCorruptionTypes() { return ContainerCorruptions.getAllParamsExcept( ContainerCorruptions.MISSING_BLOCK, ContainerCorruptions.CORRUPT_BLOCK, ContainerCorruptions.TRUNCATED_BLOCK); } - @BeforeClass - public static void init() throws Exception { + @BeforeAll + static void init() throws Exception { OzoneConfiguration ozoneConfig = new OzoneConfiguration(); // Speed up SCM closing of open container when an unhealthy replica is // reported. @@ -80,42 +79,39 @@ public static void init() throws Exception { buildCluster(ozoneConfig); } - public TestBackgroundContainerMetadataScannerIntegration( - ContainerCorruptions corruption) { - this.corruption = corruption; - logCapturer = GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME); - } - /** * {@link BackgroundContainerMetadataScanner} should detect corrupted metadata * in open or closed containers without client interaction. */ - @Test - public void testCorruptionDetected() throws Exception { + @ParameterizedTest + @MethodSource("supportedCorruptionTypes") + void testCorruptionDetected(ContainerCorruptions corruption) + throws Exception { // Write data to an open and closed container. long closedContainerID = writeDataThenCloseContainer(); - Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, - getDnContainer(closedContainerID).getContainerState()); + Container closedContainer = getDnContainer(closedContainerID); + assertEquals(State.CLOSED, closedContainer.getContainerState()); + long openContainerID = writeDataToOpenContainer(); - Assert.assertEquals(ContainerProtos.ContainerDataProto.State.OPEN, - getDnContainer(openContainerID).getContainerState()); + Container openContainer = getDnContainer(openContainerID); + assertEquals(State.OPEN, openContainer.getContainerState()); // Corrupt both containers. - corruption.applyTo(getDnContainer(closedContainerID)); - corruption.applyTo(getDnContainer(openContainerID)); + corruption.applyTo(closedContainer); + corruption.applyTo(openContainer); + // Wait for the scanner to detect corruption. - GenericTestUtils.waitFor(() -> - getDnContainer(closedContainerID).getContainerState() == - ContainerProtos.ContainerDataProto.State.UNHEALTHY, + GenericTestUtils.waitFor( + () -> closedContainer.getContainerState() == State.UNHEALTHY, 500, 5000); - GenericTestUtils.waitFor(() -> - getDnContainer(openContainerID).getContainerState() == - ContainerProtos.ContainerDataProto.State.UNHEALTHY, + GenericTestUtils.waitFor( + () -> openContainer.getContainerState() == State.UNHEALTHY, 500, 5000); // Wait for SCM to get reports of the unhealthy replicas. waitForScmToSeeUnhealthyReplica(closedContainerID); waitForScmToSeeUnhealthyReplica(openContainerID); + // Once the unhealthy replica is reported, the open container's lifecycle // state in SCM should move to closed. waitForScmToCloseContainer(openContainerID); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java index f1654615165..53407dbf570 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java @@ -45,12 +45,8 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.apache.ozone.test.LambdaTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Timeout; import java.io.File; import java.io.IOException; @@ -58,9 +54,7 @@ import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.EnumSet; import java.util.Objects; import java.util.Optional; @@ -75,17 +69,19 @@ import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * This class tests the data scanner functionality. */ +@Timeout(300) public abstract class TestContainerScannerIntegrationAbstract { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); private static MiniOzoneCluster cluster; private static OzoneClient ozClient = null; private static ObjectStore store = null; @@ -120,8 +116,8 @@ public static void buildCluster(OzoneConfiguration ozoneConfig) bucket = volume.getBucket(bucketName); } - @AfterClass - public static void shutdown() throws IOException { + @AfterAll + static void shutdown() throws IOException { if (ozClient != null) { ozClient.close(); } @@ -148,7 +144,7 @@ protected void waitForScmToCloseContainer(long containerID) throws Exception { } protected Container getDnContainer(long containerID) { - Assert.assertEquals(1, cluster.getHddsDatanodes().size()); + assertEquals(1, cluster.getHddsDatanodes().size()); HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); return oc.getContainerSet().getContainer(containerID); @@ -202,7 +198,7 @@ protected ContainerReplica getContainerReplica( ContainerID.valueOf( containerId)); // Only using a single datanode cluster. - Assert.assertEquals(1, containerReplicas.size()); + assertEquals(1, containerReplicas.size()); return containerReplicas.iterator().next(); } @@ -210,7 +206,7 @@ protected ContainerReplica getContainerReplica( @SuppressWarnings("ResultOfMethodCallIgnored") protected void readFromCorruptedKey(String keyName) throws IOException { try (OzoneInputStream key = bucket.readKey(keyName)) { - Assert.assertThrows(IOException.class, key::read); + assertThrows(IOException.class, key::read); } } @@ -233,7 +229,7 @@ protected enum ContainerCorruptions { // Fail the test. throw new UncheckedIOException(ex); } - Assert.assertFalse(chunksDir.exists()); + assertFalse(chunksDir.exists()); }, ScanResult.FailureType.MISSING_CHUNKS_DIR), MISSING_METADATA_DIR(container -> { @@ -246,13 +242,13 @@ protected enum ContainerCorruptions { // Fail the test. throw new UncheckedIOException(ex); } - Assert.assertFalse(metadataDir.exists()); + assertFalse(metadataDir.exists()); }, ScanResult.FailureType.MISSING_METADATA_DIR), MISSING_CONTAINER_FILE(container -> { File containerFile = container.getContainerFile(); - Assert.assertTrue(containerFile.delete()); - Assert.assertFalse(containerFile.exists()); + assertTrue(containerFile.delete()); + assertFalse(containerFile.exists()); }, ScanResult.FailureType.MISSING_CONTAINER_FILE), MISSING_CONTAINER_DIR(container -> { @@ -264,7 +260,7 @@ protected enum ContainerCorruptions { // Fail the test. throw new UncheckedIOException(ex); } - Assert.assertFalse(containerDir.exists()); + assertFalse(containerDir.exists()); }, ScanResult.FailureType.MISSING_CONTAINER_DIR), MISSING_BLOCK(container -> { @@ -297,7 +293,7 @@ protected enum ContainerCorruptions { Optional blockFile = Arrays.stream(Objects.requireNonNull( chunksDir.listFiles((dir, name) -> name.endsWith(".block")))) .findFirst(); - Assert.assertTrue(blockFile.isPresent()); + assertTrue(blockFile.isPresent()); corruptFile(blockFile.get()); }, ScanResult.FailureType.CORRUPT_CHUNK), @@ -307,7 +303,7 @@ protected enum ContainerCorruptions { Optional blockFile = Arrays.stream(Objects.requireNonNull( chunksDir.listFiles((dir, name) -> name.endsWith(".block")))) .findFirst(); - Assert.assertTrue(blockFile.isPresent()); + assertTrue(blockFile.isPresent()); truncateFile(blockFile.get()); }, ScanResult.FailureType.INCONSISTENT_CHUNK_LENGTH); @@ -330,27 +326,20 @@ public void applyTo(Container container) { * Check that the correct corruption type was written to the container log. */ public void assertLogged(LogCapturer logCapturer) { - Assert.assertTrue(logCapturer.getOutput() - .contains(expectedResult.toString())); + assertThat(logCapturer.getOutput(), + containsString(expectedResult.toString())); } /** * Get all container corruption types as parameters for junit 4 * parameterized tests, except the ones specified. */ - public static Collection getAllParamsExcept( + public static Set getAllParamsExcept( ContainerCorruptions... exclude) { - Collection params = new ArrayList<>(); Set includeSet = EnumSet.allOf(ContainerCorruptions.class); Arrays.asList(exclude).forEach(includeSet::remove); - - for (ContainerCorruptions c: values()) { - if (includeSet.contains(c)) { - params.add(new Object[]{c}); - } - } - return params; + return includeSet; } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java index 1430e4f9653..4b47d061e7c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java @@ -20,30 +20,30 @@ package org.apache.hadoop.ozone.dn.scanner; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; +import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.utils.ContainerLogger; import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration; import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Collection; +import static org.junit.jupiter.api.Assertions.assertEquals; + /** * Integration tests for the on demand container data scanner. This scanner * is triggered when there is an error while a client interacts with a * container. */ -@RunWith(Parameterized.class) -public class TestOnDemandContainerDataScannerIntegration +class TestOnDemandContainerDataScannerIntegration extends TestContainerScannerIntegrationAbstract { - private final ContainerCorruptions corruption; - private final GenericTestUtils.LogCapturer logCapturer; + private final GenericTestUtils.LogCapturer logCapturer = + GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME); /** The on-demand container scanner is triggered by errors on the block read @@ -57,8 +57,7 @@ public class TestOnDemandContainerDataScannerIntegration - Block checksums are verified on the client side. If there is a checksum error during read, the datanode will not learn about it. */ - @Parameterized.Parameters(name = "{0}") - public static Collection supportedCorruptionTypes() { + static Collection supportedCorruptionTypes() { return ContainerCorruptions.getAllParamsExcept( ContainerCorruptions.MISSING_METADATA_DIR, ContainerCorruptions.MISSING_CONTAINER_FILE, @@ -68,8 +67,8 @@ public static Collection supportedCorruptionTypes() { ContainerCorruptions.TRUNCATED_BLOCK); } - @BeforeClass - public static void init() throws Exception { + @BeforeAll + static void init() throws Exception { OzoneConfiguration ozoneConfig = new OzoneConfiguration(); ozoneConfig.setBoolean( ContainerScannerConfiguration.HDDS_CONTAINER_SCRUB_ENABLED, @@ -85,33 +84,28 @@ public static void init() throws Exception { buildCluster(ozoneConfig); } - public TestOnDemandContainerDataScannerIntegration( - ContainerCorruptions corruption) { - this.corruption = corruption; - logCapturer = GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME); - } - /** * {@link OnDemandContainerDataScanner} should detect corrupted blocks * in a closed container when a client reads from it. */ - @Test - public void testCorruptionDetected() throws Exception { + @ParameterizedTest + @MethodSource("supportedCorruptionTypes") + void testCorruptionDetected(ContainerCorruptions corruption) + throws Exception { String keyName = "testKey"; long containerID = writeDataThenCloseContainer(keyName); // Container corruption has not yet been introduced. - Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, - getDnContainer(containerID).getContainerState()); + Container container = getDnContainer(containerID); + assertEquals(State.CLOSED, container.getContainerState()); // Corrupt the container. - corruption.applyTo(getDnContainer(containerID)); + corruption.applyTo(container); // This method will check that reading from the corrupted key returns an // error to the client. readFromCorruptedKey(keyName); // Reading from the corrupted key should have triggered an on-demand scan // of the container, which will detect the corruption. - GenericTestUtils.waitFor(() -> - getDnContainer(containerID).getContainerState() == - ContainerProtos.ContainerDataProto.State.UNHEALTHY, + GenericTestUtils.waitFor( + () -> container.getContainerState() == State.UNHEALTHY, 500, 5000); // Wait for SCM to get a report of the unhealthy replica. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java index 37ac7c6318d..3e22c1db90d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java @@ -19,25 +19,23 @@ */ package org.apache.hadoop.ozone.dn.volume; -import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.interfaces.Container; @@ -49,73 +47,218 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.dn.DatanodeTestUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; import java.util.UUID; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Timeout; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; +import static java.util.Collections.emptyMap; +import static org.apache.commons.io.IOUtils.readFully; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * This class tests datanode can detect failed volumes. */ -@RunWith(Parameterized.class) -public class TestDatanodeHddsVolumeFailureDetection { - private boolean schemaV3; - public TestDatanodeHddsVolumeFailureDetection(boolean enableV3) { - this.schemaV3 = enableV3; +@Timeout(300) +class TestDatanodeHddsVolumeFailureDetection { + + private static final int KEY_SIZE = 128; + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void corruptChunkFile(boolean schemaV3) throws Exception { + try (MiniOzoneCluster cluster = newCluster(schemaV3)) { + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + + // write a file + String keyName = UUID.randomUUID().toString(); + long containerId = createKey(bucket, keyName); + + // corrupt chunk file by rename file->dir + HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + MutableVolumeSet volSet = oc.getVolumeSet(); + StorageVolume vol0 = volSet.getVolumesList().get(0); + HddsVolume volume = assertInstanceOf(HddsVolume.class, vol0); + Path chunksPath = Paths.get( + volume.getStorageDir().getPath(), + volume.getClusterID(), + Storage.STORAGE_DIR_CURRENT, + Storage.CONTAINER_DIR + "0", + String.valueOf(containerId), + OzoneConsts.STORAGE_DIR_CHUNKS + ); + File[] chunkFiles = chunksPath.toFile().listFiles(); + assertNotNull(chunkFiles); + + try { + for (File chunkFile : chunkFiles) { + DatanodeTestUtils.injectDataFileFailure(chunkFile); + } + + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); + + // read written file to trigger checkVolumeAsync + readKeyToTriggerCheckVolumeAsync(bucket, keyName); + + // should trigger checkVolumeAsync and + // a failed volume should be detected + DatanodeTestUtils.waitForCheckVolume(volSet, 1L); + DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); + } finally { + // restore for cleanup + DatanodeTestUtils.restoreBadVolume(vol0); + for (File chunkFile : chunkFiles) { + DatanodeTestUtils.restoreDataFileFromFailure(chunkFile); + } + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void corruptContainerFile(boolean schemaV3) throws Exception { + try (MiniOzoneCluster cluster = newCluster(schemaV3)) { + // create a container + ContainerWithPipeline container; + OzoneConfiguration conf = cluster.getConf(); + try (ScmClient scmClient = new ContainerOperationClient(conf)) { + container = scmClient.createContainer(ReplicationType.STAND_ALONE, + ReplicationFactor.ONE, OzoneConsts.OZONE); + } + + // corrupt container file by removing write permission on + // container metadata dir, since container update operation + // use a create temp & rename way, so we can't just rename + // container file to simulate corruption + HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + MutableVolumeSet volSet = oc.getVolumeSet(); + StorageVolume vol0 = volSet.getVolumesList().get(0); + Container c1 = oc.getContainerSet().getContainer( + container.getContainerInfo().getContainerID()); + File metadataDir = new File(c1.getContainerFile().getParent()); + try { + DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir); + + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); + + // close container to trigger checkVolumeAsync + assertThrows(IOException.class, c1::close); + + // should trigger CheckVolumeAsync and + // a failed volume should be detected + DatanodeTestUtils.waitForCheckVolume(volSet, 1L); + DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); + } finally { + // restore for cleanup + DatanodeTestUtils.restoreBadVolume(vol0); + DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir); + } + } } - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[]{false}, - new Object[]{true}); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void corruptDbFile(boolean schemaV3) throws Exception { + try (MiniOzoneCluster cluster = newCluster(schemaV3)) { + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + + // write a file, will create container1 + String keyName = UUID.randomUUID().toString(); + long containerId = createKey(bucket, keyName); + + // close container1 + HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + Container c1 = oc.getContainerSet().getContainer(containerId); + c1.close(); + + // create container2, and container1 is kicked out of cache + OzoneConfiguration conf = cluster.getConf(); + try (ScmClient scmClient = new ContainerOperationClient(conf)) { + ContainerWithPipeline c2 = scmClient.createContainer( + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, + OzoneConsts.OZONE); + assertEquals(c2.getContainerInfo().getState(), LifeCycleState.OPEN); + } + + // corrupt db by rename dir->file + File dbDir; + if (schemaV3) { + dbDir = new File(((KeyValueContainerData) (c1.getContainerData())) + .getDbFile().getAbsolutePath()); + } else { + File metadataDir = new File(c1.getContainerFile().getParent()); + dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB); + } + + MutableVolumeSet volSet = oc.getVolumeSet(); + StorageVolume vol0 = volSet.getVolumesList().get(0); + + try { + DatanodeTestUtils.injectDataDirFailure(dbDir); + if (schemaV3) { + // remove rocksDB from cache + DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath()); + } + + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); + + readKeyToTriggerCheckVolumeAsync(bucket, keyName); + + // should trigger CheckVolumeAsync and + // a failed volume should be detected + DatanodeTestUtils.waitForCheckVolume(volSet, 1L); + DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); + } finally { + // restore all + DatanodeTestUtils.restoreBadVolume(vol0); + DatanodeTestUtils.restoreDataDirFromFailure(dbDir); + } + } + } + } + + private static void readKeyToTriggerCheckVolumeAsync(OzoneBucket bucket, + String key) throws IOException { + try (InputStream is = bucket.readKey(key)) { + assertThrows(IOException.class, () -> readFully(is, new byte[KEY_SIZE])); + } } - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private MiniOzoneCluster cluster; - private OzoneConfiguration ozoneConfig; - private OzoneClient ozClient = null; - private ScmClient scmClient; - private ObjectStore store = null; - private OzoneVolume volume; - private OzoneBucket bucket; - private List datanodes; - - @Before - public void init() throws Exception { - ozoneConfig = new OzoneConfiguration(); + private static MiniOzoneCluster newCluster(boolean schemaV3) + throws Exception { + OzoneConfiguration ozoneConfig = new OzoneConfiguration(); ozoneConfig.set(OZONE_SCM_CONTAINER_SIZE, "1GB"); ozoneConfig.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, 0, StorageUnit.MB); - ozoneConfig.setInt(OZONE_REPLICATION, ReplicationFactor.ONE.getValue()); + ozoneConfig.setInt(OZONE_REPLICATION, 1); // keep the cache size = 1, so we could trigger io exception on // reading on-disk db instance ozoneConfig.setInt(OZONE_CONTAINER_CACHE_SIZE, 1); @@ -129,208 +272,28 @@ public void init() throws Exception { dnConf.setFailedDataVolumesTolerated(1); dnConf.setDiskCheckMinGap(Duration.ofSeconds(5)); ozoneConfig.setFromObject(dnConf); - cluster = MiniOzoneCluster.newBuilder(ozoneConfig) + MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(ozoneConfig) .setNumDatanodes(1) .setNumDataVolumes(1) .build(); cluster.waitForClusterToBeReady(); - cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 30000); - - ozClient = OzoneClientFactory.getRpcClient(ozoneConfig); - store = ozClient.getObjectStore(); - scmClient = new ContainerOperationClient(ozoneConfig); - - String volumeName = UUID.randomUUID().toString(); - store.createVolume(volumeName); - volume = store.getVolume(volumeName); - - String bucketName = UUID.randomUUID().toString(); - volume.createBucket(bucketName); - bucket = volume.getBucket(bucketName); - - datanodes = cluster.getHddsDatanodes(); - } - - @After - public void shutdown() throws IOException { - IOUtils.closeQuietly(scmClient); - if (ozClient != null) { - ozClient.close(); - } - if (cluster != null) { - cluster.shutdown(); - } - } - - @Test - public void testHddsVolumeFailureOnChunkFileCorrupt() throws Exception { - // write a file - String keyName = UUID.randomUUID().toString(); - String value = "sample value"; - OzoneOutputStream out = bucket.createKey(keyName, - value.getBytes(UTF_8).length, RATIS, - ONE, new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - OzoneKey key = bucket.getKey(keyName); - Assert.assertEquals(keyName, key.getName()); - - // corrupt chunk file by rename file->dir - HddsDatanodeService dn = datanodes.get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - MutableVolumeSet volSet = oc.getVolumeSet(); - StorageVolume vol0 = volSet.getVolumesList().get(0); - Assert.assertTrue(vol0 instanceof HddsVolume); - File clusterDir = DatanodeTestUtils.getHddsVolumeClusterDir( - (HddsVolume) vol0); - File currentDir = new File(clusterDir, Storage.STORAGE_DIR_CURRENT); - File containerTopDir = new File(currentDir, Storage.CONTAINER_DIR + "0"); - File containerDir = new File(containerTopDir, "1"); - File chunksDir = new File(containerDir, OzoneConsts.STORAGE_DIR_CHUNKS); - File[] chunkFiles = chunksDir.listFiles(); - Assert.assertNotNull(chunkFiles); - for (File chunkFile : chunkFiles) { - DatanodeTestUtils.injectDataFileFailure(chunkFile); - } - - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - DatanodeTestUtils.simulateBadVolume(vol0); - - // read written file to trigger checkVolumeAsync - OzoneInputStream is = bucket.readKey(keyName); - byte[] fileContent = new byte[value.getBytes(UTF_8).length]; - - try { - is.read(fileContent); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e instanceof IOException); - } finally { - is.close(); - } + cluster.waitForPipelineTobeReady(ReplicationFactor.ONE, 30000); - // should trigger checkVolumeAsync and - // a failed volume should be detected - DatanodeTestUtils.waitForCheckVolume(volSet, 1L); - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); - - // restore for cleanup - DatanodeTestUtils.restoreBadVolume(vol0); - for (File chunkFile : chunkFiles) { - DatanodeTestUtils.restoreDataFileFromFailure(chunkFile); - } + return cluster; } - @Test - public void testHddsVolumeFailureOnContainerFileCorrupt() throws Exception { - // create a container - ContainerWithPipeline container = scmClient.createContainer(HddsProtos - .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor - .ONE, OzoneConsts.OZONE); - - // corrupt container file by removing write permission on - // container metadata dir, since container update operation - // use a create temp & rename way, so we can't just rename - // container file to simulate corruption - HddsDatanodeService dn = datanodes.get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - MutableVolumeSet volSet = oc.getVolumeSet(); - StorageVolume vol0 = volSet.getVolumesList().get(0); - Container c1 = oc.getContainerSet().getContainer(container. - getContainerInfo().getContainerID()); - File metadataDir = new File(c1.getContainerFile().getParent()); - DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir); - - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - DatanodeTestUtils.simulateBadVolume(vol0); - - // close container to trigger checkVolumeAsync - try { - c1.close(); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e instanceof IOException); + private static long createKey(OzoneBucket bucket, String key) + throws IOException { + byte[] bytes = RandomUtils.nextBytes(KEY_SIZE); + RatisReplicationConfig replication = + RatisReplicationConfig.getInstance(ReplicationFactor.ONE); + try (OutputStream out = bucket.createKey(key, bytes.length, replication, + emptyMap())) { + out.write(bytes); } - - // should trigger CheckVolumeAsync and - // a failed volume should be detected - DatanodeTestUtils.waitForCheckVolume(volSet, 1L); - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); - - // restore for cleanup - DatanodeTestUtils.restoreBadVolume(vol0); - DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir); + OzoneKeyDetails keyDetails = bucket.getKey(key); + assertEquals(key, keyDetails.getName()); + return keyDetails.getOzoneKeyLocations().get(0).getContainerID(); } - @Test - public void testHddsVolumeFailureOnDbFileCorrupt() throws Exception { - // write a file, will create container1 - String keyName = UUID.randomUUID().toString(); - String value = "sample value"; - OzoneOutputStream out = bucket.createKey(keyName, - value.getBytes(UTF_8).length, RATIS, - ONE, new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - OzoneKey key = bucket.getKey(keyName); - Assert.assertEquals(keyName, key.getName()); - - // close container1 - HddsDatanodeService dn = datanodes.get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - Container c1 = oc.getContainerSet().getContainer(1); - c1.close(); - - // create container2, and container1 is kicked out of cache - ContainerWithPipeline c2 = scmClient.createContainer(HddsProtos - .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor - .ONE, OzoneConsts.OZONE); - Assert.assertTrue(c2.getContainerInfo().getState() - .equals(HddsProtos.LifeCycleState.OPEN)); - - // corrupt db by rename dir->file - File dbDir; - if (schemaV3) { - dbDir = new File(((KeyValueContainerData)(c1.getContainerData())) - .getDbFile().getAbsolutePath()); - } else { - File metadataDir = new File(c1.getContainerFile().getParent()); - dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB); - } - DatanodeTestUtils.injectDataDirFailure(dbDir); - if (schemaV3) { - // remove rocksDB from cache - DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath()); - } - - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - MutableVolumeSet volSet = oc.getVolumeSet(); - StorageVolume vol0 = volSet.getVolumesList().get(0); - DatanodeTestUtils.simulateBadVolume(vol0); - - // read written file to trigger checkVolumeAsync - OzoneInputStream is = bucket.readKey(keyName); - byte[] fileContent = new byte[value.getBytes(UTF_8).length]; - - try { - is.read(fileContent); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e instanceof IOException); - } finally { - is.close(); - } - - // should trigger CheckVolumeAsync and - // a failed volume should be detected - DatanodeTestUtils.waitForCheckVolume(volSet, 1L); - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); - - // restore all - DatanodeTestUtils.restoreBadVolume(vol0); - DatanodeTestUtils.restoreDataDirFromFailure(dbDir); - } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucket.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucket.java new file mode 100644 index 00000000000..00b92296cf1 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucket.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om; + +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.KeyInputStream; +import org.apache.hadoop.ozone.container.ContainerTestHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; +import static org.apache.commons.lang3.RandomStringUtils.randomNumeric; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +/** + * Wrapper for {@code OzoneBucket} for testing. Can create random keys, + * verify content, etc. + */ +public final class TestBucket { + + private final OzoneBucket bucket; + + private TestBucket(OzoneBucket bucket) { + this.bucket = bucket; + } + + public static Builder newBuilder(OzoneClient client) { + return new Builder(client); + } + + public OzoneBucket delegate() { + return bucket; + } + + public KeyInputStream getKeyInputStream(String keyName) throws IOException { + return (KeyInputStream) bucket + .readKey(keyName).getInputStream(); + } + + public byte[] writeKey(String keyName, int dataLength) throws Exception { + ReplicationConfig repConfig = RatisReplicationConfig.getInstance(THREE); + return writeKey(keyName, repConfig, dataLength); + } + + public byte[] writeKey(String key, ReplicationConfig repConfig, int len) + throws Exception { + byte[] inputData = ContainerTestHelper + .getFixedLengthString(UUID.randomUUID().toString(), len) + .getBytes(UTF_8); + + writeKey(key, repConfig, inputData); + + return inputData; + } + + public void writeKey(String key, ReplicationConfig repConfig, + byte[] inputData) throws IOException { + try (OutputStream out = bucket.createKey(key, 0, repConfig, emptyMap())) { + out.write(inputData); + } + } + + public byte[] writeRandomBytes(String keyName, int dataLength) + throws Exception { + ReplicationConfig repConfig = RatisReplicationConfig.getInstance(THREE); + return writeRandomBytes(keyName, repConfig, dataLength); + } + + public byte[] writeRandomBytes(String key, ReplicationConfig repConfig, + int len) throws Exception { + byte[] inputData = new byte[len]; + ThreadLocalRandom.current().nextBytes(inputData); + + writeKey(key, repConfig, inputData); + return inputData; + } + + public void validateData(byte[] inputData, int offset, byte[] readData) { + int readDataLen = readData.length; + byte[] expectedData = new byte[readDataLen]; + System.arraycopy(inputData, offset, expectedData, 0, readDataLen); + + assertArrayEquals(expectedData, readData); + } + + /** + * Builder for {@code TestBucket}. + */ + public static class Builder { + private final OzoneClient client; + private OzoneVolume volume; + private String volumeName; + private String bucketName; + + Builder(OzoneClient client) { + this.client = client; + } + + public TestBucket build() throws IOException { + ObjectStore objectStore = client.getObjectStore(); + if (volume == null) { // TODO add setVolume + if (volumeName == null) { // TODO add setVolumeName + volumeName = "vol" + randomNumeric(10); + } + objectStore.createVolume(volumeName); + volume = objectStore.getVolume(volumeName); + } + if (bucketName == null) { // TODO add setBucketName + bucketName = "bucket" + randomNumeric(10); + } + volume.createBucket(bucketName); + return new TestBucket(volume.getBucket(bucketName)); + } + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 16b3b4b79fe..f56a95d30a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -23,41 +23,39 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer; import org.apache.ozone.test.LambdaTestUtils; -import org.junit.Test; -import org.junit.Before; -import org.junit.After; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Collection; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Set; import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; import static org.apache.hadoop.ozone.om.OMUpgradeTestUtils.waitForFinalization; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Upgrade testing for Bucket Layout Feature. @@ -68,50 +66,30 @@ *

* 2. Post-Finalize: OM should allow creation of buckets with new bucket * layouts. + * + * Test cases are run on the same single cluster. Order is important, + * because "upgrade" changes its state. Test cases are therefore assigned to + * 3 groups: before, during and after upgrade. */ -@RunWith(Parameterized.class) -public class TestOMBucketLayoutUpgrade { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(300000)); +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@Timeout(300) +class TestOMBucketLayoutUpgrade { + + private static final int PRE_UPGRADE = 100; + private static final int DURING_UPGRADE = 200; + private static final int POST_UPGRADE = 300; + private MiniOzoneHAClusterImpl cluster; private OzoneManager ozoneManager; - private ClientProtocol clientProtocol; private static final String VOLUME_NAME = "vol-" + UUID.randomUUID(); - private int fromLayoutVersion; + private final int fromLayoutVersion = INITIAL_VERSION.layoutVersion(); private OzoneManagerProtocol omClient; - private static final Logger LOG = - LoggerFactory.getLogger(TestOMBucketLayoutUpgrade.class); private OzoneClient client; - /** - * Defines a "from" layout version to finalize from. - * - * @return - */ - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][]{ - {INITIAL_VERSION} - }); - } - - - public TestOMBucketLayoutUpgrade(OMLayoutFeature fromVersion) { - this.fromLayoutVersion = fromVersion.layoutVersion(); - } - - /** - * Create a MiniDFSCluster for testing. - */ - @Before - public void setup() throws Exception { - org.junit.Assume.assumeTrue("Check if there is need to finalize.", - maxLayoutVersion() > fromLayoutVersion); - + @BeforeAll + void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String omServiceId = UUID.randomUUID().toString(); cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) @@ -125,10 +103,8 @@ public void setup() throws Exception { cluster.waitForClusterToBeReady(); ozoneManager = cluster.getOzoneManager(); - client = OzoneClientFactory.getRpcClient(omServiceId, conf); - ObjectStore objectStore = client.getObjectStore(); - clientProtocol = objectStore.getClientProxy(); - omClient = clientProtocol.getOzoneManagerClient(); + client = cluster.newClient(); + omClient = client.getObjectStore().getClientProxy().getOzoneManagerClient(); // create sample volume. omClient.createVolume( @@ -139,102 +115,78 @@ public void setup() throws Exception { .build()); } - /** - * Shutdown MiniDFSCluster. - */ - @After - public void shutdown() { + @AfterAll + void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); } } - /** - * Tests that OM blocks all requests to create any buckets with a new bucket - * layout. - * - * @throws Exception - */ @Test - public void testCreateBucketWithBucketLayoutsDuringUpgrade() - throws Exception { - // Assert OM layout version is 'fromLayoutVersion' on deploy. + @Order(PRE_UPGRADE) + void omLayoutBeforeUpgrade() throws IOException { assertEquals(fromLayoutVersion, ozoneManager.getVersionManager().getMetadataLayoutVersion()); assertNull(ozoneManager.getMetadataManager().getMetaTable() .get(LAYOUT_VERSION_KEY)); + } - // Test bucket creation with new bucket layouts. - // FSO and OBS bucket creation should fail. - verifyBucketCreationBlockedWithNewLayouts(); - - // Bucket creation with LEGACY layout should succeed in Pre-Finalized state. - LOG.info("Creating legacy bucket during Pre-Finalize"); - verifyBucketCreationWithLayout(new BucketLayout[]{BucketLayout.LEGACY}); - - // Finalize the cluster upgrade. - finalizeUpgrade(); - - // Cluster upgrade is now complete, - // Bucket creation should now succeed with all layouts. - verifyBucketCreationWithLayout(new BucketLayout[]{ - BucketLayout.LEGACY, - BucketLayout.FILE_SYSTEM_OPTIMIZED, - BucketLayout.OBJECT_STORE - }); + /** + * Tests that OM blocks all requests to create any buckets with a new bucket + * layout. + */ + @Order(PRE_UPGRADE) + @ParameterizedTest + @MethodSource("layoutsNotAllowedBeforeUpgrade") + void blocksNewLayoutBeforeUpgrade(BucketLayout layout) { + OMException e = assertThrows(OMException.class, + () -> createBucketWithLayout(layout)); + assertEquals(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, e.getResult()); } + @Order(PRE_UPGRADE) + @Test + void allowsLegacyBucketBeforeUpgrade() throws Exception { + assertCreateBucketWithLayout(BucketLayout.LEGACY); + } /** - * Tests that OM allows bucket creation with given bucket layouts. - * - * @param bucketLayouts bucket layouts to test - * @throws Exception if any + * Complete the cluster upgrade. */ - private void verifyBucketCreationWithLayout(BucketLayout[] bucketLayouts) - throws Exception { - String bucketName; - for (BucketLayout layout : bucketLayouts) { - LOG.info("Creating bucket with layout {} after OM finalization", layout); + @Test + @Order(DURING_UPGRADE) + void finalizeUpgrade() throws Exception { + UpgradeFinalizer.StatusAndMessages response = + omClient.finalizeUpgrade("finalize-test"); + System.out.println("Finalization Messages : " + response.msgs()); - bucketName = createBucketWithLayout(layout); + waitForFinalization(omClient); - // Make sure the bucket exists in the bucket table with the - // expected layout. - assertEquals( - omClient.getBucketInfo(VOLUME_NAME, bucketName).getBucketName(), - bucketName); - assertEquals( - omClient.getBucketInfo(VOLUME_NAME, bucketName).getBucketLayout(), - layout); - } + final String expectedVersion = String.valueOf(maxLayoutVersion()); + LambdaTestUtils.await(30000, 3000, + () -> expectedVersion.equals( + ozoneManager.getMetadataManager().getMetaTable() + .get(LAYOUT_VERSION_KEY))); } - /** - * Tests that OM blocks all requests to create any buckets with a new bucket - * layout. - * - * @throws Exception if any - */ - private void verifyBucketCreationBlockedWithNewLayouts() throws Exception { - BucketLayout[] bucketLayouts = new BucketLayout[]{ - BucketLayout.OBJECT_STORE, - BucketLayout.FILE_SYSTEM_OPTIMIZED, - }; + @Order(POST_UPGRADE) + @ParameterizedTest + @EnumSource + void allowsBucketCreationWithAnyLayoutAfterUpgrade(BucketLayout layout) + throws Exception { + assertCreateBucketWithLayout(layout); + } - for (BucketLayout layout : bucketLayouts) { - try { - LOG.info("Creating bucket with layout {} during Pre-Finalize", layout); - createBucketWithLayout(layout); - fail("Expected to fail creating bucket with layout " + layout); - } catch (OMException e) { - // Expected exception. - assertEquals( - OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, - e.getResult()); - } - } + private void assertCreateBucketWithLayout(BucketLayout layout) + throws Exception { + String bucketName = createBucketWithLayout(layout); + + // Make sure the bucket exists in the bucket table with the + // expected layout. + OmBucketInfo bucketInfo = omClient.getBucketInfo(VOLUME_NAME, bucketName); + assertEquals(bucketName, bucketInfo.getBucketName()); + assertEquals(layout, bucketInfo.getBucketLayout()); } /** @@ -257,22 +209,7 @@ private String createBucketWithLayout(BucketLayout bucketLayout) return bucketName; } - /** - * Complete the cluster upgrade. - * - * @throws Exception if upgrade fails. - */ - private void finalizeUpgrade() throws Exception { - UpgradeFinalizer.StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - - waitForFinalization(omClient); - - LambdaTestUtils.await(30000, 3000, () -> { - String lvString = ozoneManager.getMetadataManager().getMetaTable() - .get(LAYOUT_VERSION_KEY); - return maxLayoutVersion() == Integer.parseInt(lvString); - }); + private static Set layoutsNotAllowedBeforeUpgrade() { + return EnumSet.complementOf(EnumSet.of(BucketLayout.LEGACY)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index e4da200d9b3..1e657d0ea78 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -1,5 +1,3 @@ -package org.apache.hadoop.ozone.om; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this @@ -17,6 +15,7 @@ * the License. * */ +package org.apache.hadoop.ozone.om; import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; import static org.apache.hadoop.ozone.om.OMUpgradeTestUtils.assertClusterPrepared; @@ -24,189 +23,94 @@ import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion; import static org.apache.ozone.test.GenericTestUtils.waitFor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; -import java.util.Arrays; -import java.util.Collection; +import java.io.IOException; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages; -import org.apache.ozone.test.LambdaTestUtils; import org.apache.ratis.util.LifeCycle; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Test; /** * Tests for OM upgrade finalization. + * TODO: can be merged into class with other OM tests with per-method cluster */ -@RunWith(Parameterized.class) -public class TestOMUpgradeFinalization { +class TestOMUpgradeFinalization { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(300000)); - private MiniOzoneHAClusterImpl cluster; - private OzoneManager ozoneManager; - private ClientProtocol clientProtocol; - private int fromLayoutVersion; - private OzoneClient client; - - /** - * Defines a "from" layout version to finalize from. - * - * @return - */ - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][]{ - {INITIAL_VERSION}, - }); - } - - - public TestOMUpgradeFinalization(OMLayoutFeature fromVersion) { - this.fromLayoutVersion = fromVersion.layoutVersion(); + @Test + void testOMUpgradeFinalizationWithOneOMDown() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) { + cluster.waitForClusterToBeReady(); + + try (OzoneClient client = cluster.newClient()) { + List runningOms = cluster.getOzoneManagersList(); + final int shutdownOMIndex = 2; + OzoneManager downedOM = cluster.getOzoneManager(shutdownOMIndex); + cluster.stopOzoneManager(shutdownOMIndex); + assertFalse(downedOM.isRunning()); + assertEquals(runningOms.remove(shutdownOMIndex), downedOM); + + OzoneManagerProtocol omClient = client.getObjectStore().getClientProxy() + .getOzoneManagerClient(); + // Have to do a "prepare" operation to get rid of the logs in the active + // OMs. + long prepareIndex = omClient.prepareOzoneManager(120L, 5L); + assertClusterPrepared(prepareIndex, runningOms); + + omClient.cancelOzoneManagerPrepare(); + StatusAndMessages response = + omClient.finalizeUpgrade("finalize-test"); + System.out.println("Finalization Messages : " + response.msgs()); + + waitForFinalization(omClient); + cluster.restartOzoneManager(downedOM, true); + + OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer() + .getOmStateMachine(); + try { + waitFor(() -> omStateMachine.getLifeCycleState().isPausingOrPaused(), + 1000, 60000); + } catch (TimeoutException timeEx) { + assertEquals(LifeCycle.State.RUNNING, + omStateMachine.getLifeCycle().getCurrentState()); + } + + waitFor(() -> !omStateMachine.getLifeCycle().getCurrentState() + .isPausingOrPaused(), 1000, 60000); + + assertEquals(maxLayoutVersion(), + downedOM.getVersionManager().getMetadataLayoutVersion()); + String lvString = downedOM.getMetadataManager().getMetaTable() + .get(LAYOUT_VERSION_KEY); + assertNotNull(lvString); + assertEquals(maxLayoutVersion(), Integer.parseInt(lvString)); + } + } } - /** - * Create a MiniDFSCluster for testing. - */ - @Before - public void setup() throws Exception { - - org.junit.Assume.assumeTrue("Check if there is need to finalize.", - maxLayoutVersion() > fromLayoutVersion); - - OzoneConfiguration conf = new OzoneConfiguration(); - String omServiceId = UUID.randomUUID().toString(); - cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + private static MiniOzoneHAClusterImpl newCluster(OzoneConfiguration conf) + throws IOException { + return (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(UUID.randomUUID().toString()) .setScmId(UUID.randomUUID().toString()) - .setOMServiceId(omServiceId) + .setOMServiceId(UUID.randomUUID().toString()) .setNumOfOzoneManagers(3) .setNumDatanodes(1) - .setOmLayoutVersion(fromLayoutVersion) + .setOmLayoutVersion(INITIAL_VERSION.layoutVersion()) .build(); - - cluster.waitForClusterToBeReady(); - ozoneManager = cluster.getOzoneManager(); - client = OzoneClientFactory.getRpcClient(omServiceId, conf); - ObjectStore objectStore = client.getObjectStore(); - clientProtocol = objectStore.getClientProxy(); - } - - /** - * Shutdown MiniDFSCluster. - */ - @After - public void shutdown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } } - /** - * Currently this is a No-Op finalization since there is only one layout - * version in OM. But this test is expected to remain consistent when a - * new version is added. - */ - @Test - public void testOmFinalization() throws Exception { - // Assert OM Layout Version is 'fromLayoutVersion' on deploy. - assertEquals(fromLayoutVersion, - ozoneManager.getVersionManager().getMetadataLayoutVersion()); - assertNull(ozoneManager.getMetadataManager() - .getMetaTable().get(LAYOUT_VERSION_KEY)); - - OzoneManagerProtocol omClient = clientProtocol.getOzoneManagerClient(); - StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - - waitForFinalization(omClient); - - LambdaTestUtils.await(30000, 3000, () -> { - String lvString = ozoneManager.getMetadataManager().getMetaTable() - .get(LAYOUT_VERSION_KEY); - return maxLayoutVersion() == Integer.parseInt(lvString); - }); - } - - @Test - public void testOmFinalizationWithOneOmDown() throws Exception { - - List runningOms = cluster.getOzoneManagersList(); - final int shutdownOMIndex = 2; - OzoneManager downedOM = cluster.getOzoneManager(shutdownOMIndex); - cluster.stopOzoneManager(shutdownOMIndex); - Assert.assertFalse(downedOM.isRunning()); - Assert.assertEquals(runningOms.remove(shutdownOMIndex), downedOM); - - OzoneManagerProtocol omClient = clientProtocol.getOzoneManagerClient(); - // Have to do a "prepare" operation to get rid of the logs in the active - // OMs. - long prepareIndex = omClient.prepareOzoneManager(120L, 5L); - assertClusterPrepared(prepareIndex, runningOms); - - omClient.cancelOzoneManagerPrepare(); - StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - - waitForFinalization(omClient); - cluster.restartOzoneManager(downedOM, true); - - try { - waitFor(() -> downedOM.getOmRatisServer() - .getOmStateMachine().getLifeCycleState().isPausingOrPaused(), - 1000, 60000); - } catch (TimeoutException timeEx) { - LifeCycle.State state = downedOM.getOmRatisServer() - .getOmStateMachine().getLifeCycle().getCurrentState(); - if (state != LifeCycle.State.RUNNING) { - Assert.fail("OM State Machine State expected to be in RUNNING state."); - } - } - - waitFor(() -> { - LifeCycle.State lifeCycleState = downedOM.getOmRatisServer() - .getOmStateMachine().getLifeCycle().getCurrentState(); - return !lifeCycleState.isPausingOrPaused(); - }, 1000, 60000); - - - assertEquals(maxLayoutVersion(), - ozoneManager.getVersionManager().getMetadataLayoutVersion()); - String lvString = ozoneManager.getMetadataManager().getMetaTable() - .get(LAYOUT_VERSION_KEY); - assertNotNull(lvString); - assertEquals(maxLayoutVersion(), - Integer.parseInt(lvString)); - } } From c65da9e267e61e2d96a3f657c8bb292b1401fdb9 Mon Sep 17 00:00:00 2001 From: Aryan Gupta <44232823+aryangupta1998@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:50:55 +0530 Subject: [PATCH 35/51] HDDS-9542. Ozone debug chunkinfo command shows incorrect number of entries. (#5703) --- .../hadoop/hdds/scm/XceiverClientGrpc.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index 32767c77eca..5f8502d1940 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -292,23 +292,25 @@ public ContainerCommandResponseProto sendCommand( Thread.currentThread().interrupt(); } } - try { - for (Map.Entry > entry : futureHashMap.entrySet()) { + try { responseProtoHashMap.put(entry.getKey(), entry.getValue().get()); - } - } catch (InterruptedException e) { - LOG.error("Command execution was interrupted."); - // Re-interrupt the thread while catching InterruptedException - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - String message = "Failed to execute command {}."; - if (LOG.isDebugEnabled()) { - LOG.debug(message, processForDebug(request), e); - } else { - LOG.error(message + " Exception Class: {}, Exception Message: {}", - request.getCmdType(), e.getClass().getName(), e.getMessage()); + } catch (InterruptedException e) { + LOG.error("Command execution was interrupted."); + // Re-interrupt the thread while catching InterruptedException + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + String message = + "Failed to execute command {} on datanode " + entry.getKey() + .getHostName(); + if (LOG.isDebugEnabled()) { + LOG.debug(message, processForDebug(request), e); + } else { + LOG.error(message + " Exception Class: {}, Exception Message: {}", + request.getCmdType(), e.getClass().getName(), e.getMessage()); + } } } return responseProtoHashMap; From 3d64e2acd8b8a3ddbabd0821e356623ee6aa3d1f Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Thu, 30 Nov 2023 13:07:19 +0000 Subject: [PATCH 36/51] HDDS-9594. Make the number of containers logged configurable in DatanodeAdminMonitorImpl (#5701) --- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 ++++ .../common/src/main/resources/ozone-default.xml | 11 +++++++++++ .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 11 +++++++---- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index a2c00d5c214..60d4ac6ae1e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -483,6 +483,10 @@ public final class ScmConfigKeys { "ozone.scm.datanode.admin.monitor.interval"; public static final String OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT = "30s"; + public static final String OZONE_SCM_DATANODE_ADMIN_MONITOR_LOGGING_LIMIT = + "ozone.scm.datanode.admin.monitor.logging.limit"; + public static final int + OZONE_SCM_DATANODE_ADMIN_MONITOR_LOGGING_LIMIT_DEFAULT = 1000; public static final String OZONE_SCM_INFO_WAIT_DURATION = "ozone.scm.info.wait.duration"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 257fcbb77ea..3eaba5a5c6a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3218,6 +3218,17 @@ if they have completed. + + ozone.scm.datanode.admin.monitor.logging.limit + 1000 + SCM + + When a node is checked for decommission or maintenance, this setting + controls how many degraded containers are logged on each pass. The limit + is applied separately for each type of container, ie under-replicated and + unhealthy will each have their own limit. + + ozone.client.list.trash.keys.max 1000 diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index 59948c0ce85..693a3474def 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -92,7 +93,7 @@ public class DatanodeAdminMonitorImpl implements DatanodeAdminMonitor { LoggerFactory.getLogger(DatanodeAdminMonitorImpl.class); // The number of containers for each of under replicated and unhealthy // that will be logged in detail each time a node is checked. - private static final int CONTAINER_DETAILS_LOGGING_LIMIT = 5; + private final int containerDetailsLoggingLimit; public DatanodeAdminMonitorImpl( OzoneConfiguration conf, @@ -103,7 +104,9 @@ public DatanodeAdminMonitorImpl( this.eventQueue = eventQueue; this.nodeManager = nodeManager; this.replicationManager = replicationManager; - + containerDetailsLoggingLimit = conf.getInt( + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_LOGGING_LIMIT, + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_LOGGING_LIMIT_DEFAULT); containerStateByHost = new HashMap<>(); } @@ -364,7 +367,7 @@ private boolean checkContainersReplicatedOnNode(DatanodeDetails dn) if (LOG.isDebugEnabled()) { underReplicatedIDs.add(cid); } - if (underReplicated < CONTAINER_DETAILS_LOGGING_LIMIT + if (underReplicated < containerDetailsLoggingLimit || LOG.isDebugEnabled()) { LOG.info("Under Replicated Container {} {}; {}", cid, replicaSet, replicaDetails(replicaSet.getReplicas())); @@ -389,7 +392,7 @@ private boolean checkContainersReplicatedOnNode(DatanodeDetails dn) if (LOG.isDebugEnabled()) { unhealthyIDs.add(cid); } - if (unhealthy < CONTAINER_DETAILS_LOGGING_LIMIT + if (unhealthy < containerDetailsLoggingLimit || LOG.isDebugEnabled()) { LOG.info("Unhealthy Container {} {}; {}", cid, replicaSet, replicaDetails(replicaSet.getReplicas())); From 911ee464e2087076f00bc1710281b7228ac98bff Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Thu, 30 Nov 2023 20:12:02 +0530 Subject: [PATCH 37/51] HDDS-9763. Over Replication Check of all UNHEALTHY replicas is broken (#5678) --- .../RatisContainerReplicaCount.java | 4 +- .../health/RatisReplicationCheckHandler.java | 24 +++- ...RatisUnhealthyReplicationCheckHandler.java | 76 +++---------- .../TestRatisContainerReplicaCount.java | 5 +- .../TestRatisOverReplicationHandler.java | 33 ++++++ .../replication/TestReplicationManager.java | 72 ++++++++---- .../TestRatisReplicationCheckHandler.java | 76 +++++++++++++ ...RatisUnhealthyReplicationCheckHandler.java | 106 +++++++++--------- 8 files changed, 254 insertions(+), 142 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java index c516bc73a07..bec3b1090e4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java @@ -559,15 +559,13 @@ public boolean isOverReplicated(boolean includePendingDelete) { * A container is safely over replicated if: * 1. It is over replicated. * 2. Has at least replication factor number of matching replicas. - * 3. # matching replicas - replication factor >= pending deletes. */ public boolean isSafelyOverReplicated() { if (!isOverReplicated(true)) { return false; } - return getMatchingReplicaCount() >= repFactor && - getMatchingReplicaCount() - repFactor >= inFlightDel; + return getMatchingReplicaCount() >= repFactor; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java index 40531cb04fc..cf7ab32b780 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java @@ -46,6 +46,15 @@ * and current replica details, along with replicas pending add and delete, * this class will return a ContainerHealthResult indicating if the container * is healthy, or under / over replicated etc. + *

+ * For example, this class handles: + *

    + *
  • A CLOSED container with 4 CLOSED replicas.
    • + *
  • A CLOSED container with 1 CLOSED replica.
    • + *
  • A CLOSED container with 3 CLOSED and 1 UNHEALTHY replica. Or 3 + * CLOSED and 1 QUASI_CLOSED replica with incorrect sequence ID.
    • + *
  • etc.
    • + *
*/ public class RatisReplicationCheckHandler extends AbstractCheck { public static final Logger LOG = @@ -133,7 +142,8 @@ public boolean handle(ContainerCheckRequest request) { ContainerHealthResult.OverReplicatedHealthResult overHealth = ((ContainerHealthResult.OverReplicatedHealthResult) health); if (!overHealth.isReplicatedOkAfterPending() && - !overHealth.hasMismatchedReplicas()) { + !overHealth.hasMismatchedReplicas() && + overHealth.isSafelyOverReplicated()) { /* A mis matched replica is one whose state does not match the container's state and the state is not UNHEALTHY. @@ -145,9 +155,12 @@ public boolean handle(ContainerCheckRequest request) { request.getReplicationQueue().enqueue(overHealth); } LOG.debug("Container {} is Over Replicated. isReplicatedOkAfterPending" + - " is [{}]. hasMismatchedReplicas is [{}]", container, + " is [{}]. hasMismatchedReplicas is [{}]. " + + "isSafelyOverReplicated is [{}].", + container, overHealth.isReplicatedOkAfterPending(), - overHealth.hasMismatchedReplicas()); + overHealth.hasMismatchedReplicas(), + overHealth.isSafelyOverReplicated()); return true; } @@ -207,8 +220,9 @@ public ContainerHealthResult checkHealth(ContainerCheckRequest request) { When checking for over replication, consider UNHEALTHY replicas. This means that other than checking over replication of healthy replicas (such as 4 CLOSED replicas of a CLOSED container), we're also checking for an excess - of UNHEALTHY replicas (such as 3 CLOSED and 1 UNHEALTHY replicas of a - CLOSED container). + of unhealthy replicas (such as 3 CLOSED and 1 UNHEALTHY replicas of a + CLOSED container, or 3 CLOSED and 1 QUASI_CLOSED with incorrect sequence + ID for a CLOSED container). */ RatisContainerReplicaCount consideringUnhealthy = new RatisContainerReplicaCount(container, replicas, replicaPendingOps, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java index a805f184801..3995c3c451d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java @@ -31,18 +31,17 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; /** - * This class handles UNHEALTHY RATIS replicas. Its responsibilities include: - *
    - *
  • If only unhealthy replicas exist, check if there are replication - * factor number of replicas and queue them for under/over replication if - * needed. - *
    • - *
  • If there is a mixture of unhealthy and healthy containers this - * handler will only be called if there is no under or over replication after - * excluding the empty containers. If there is under or over replication the - * ECReplicationCheckHandler will take care of it first. - *
    • - *
+ * This class handles RATIS containers which only have replicas in UNHEALTHY + * state, or CLOSED containers with replicas in QUASI_CLOSED state having + * less sequence ID. There are no other replicas. This class ensures that + * such containers have replication factor number of UNHEALTHY/QUASI_CLOSED + * replicas. + *

+ * For example, if a CLOSED container with replication factor 3 has 4 UNHEALTHY + * replicas, then it's called over replicated and 1 UNHEALTHY replica must be + * deleted. On the other hand, if it has only 2 UNHEALTHY replicas, it's + * under replicated and 1 more replica should be created. + *

*/ public class RatisUnhealthyReplicationCheckHandler extends AbstractCheck { public static final Logger LOG = LoggerFactory.getLogger( @@ -57,24 +56,15 @@ public boolean handle(ContainerCheckRequest request) { ReplicationManagerReport report = request.getReport(); ContainerInfo container = request.getContainerInfo(); - /* - First, verify there's perfect replication without considering UNHEALTHY - replicas. If not, we return false. Replication issues without UNHEALTHY - replicas should be solved first. - */ - if (!verifyPerfectReplication(request)) { - return false; - } - - // Now, consider UNHEALTHY replicas when calculating replication status RatisContainerReplicaCount replicaCount = getReplicaCount(request); - if (replicaCount.getUnhealthyReplicaCount() == 0) { - LOG.debug("No UNHEALTHY replicas are present for container {} with " + - "replicas [{}].", container, request.getContainerReplicas()); + if (replicaCount.getHealthyReplicaCount() > 0 || + replicaCount.getUnhealthyReplicaCount() == 0) { + LOG.debug("Not handling container {} with replicas [{}].", container, + request.getContainerReplicas()); return false; } else { - LOG.debug("Container {} has UNHEALTHY replicas. Checking its " + - "replication status.", container); + LOG.info("Container {} has unhealthy replicas [{}]. Checking its " + + "replication status.", container, replicaCount.getReplicas()); report.incrementAndSample(ReplicationManagerReport.HealthState.UNHEALTHY, container.containerID()); } @@ -114,8 +104,7 @@ public boolean handle(ContainerCheckRequest request) { overHealth.isReplicatedOkAfterPending(), overHealth.hasMismatchedReplicas()); - if (!overHealth.isReplicatedOkAfterPending() && - overHealth.isSafelyOverReplicated()) { + if (!overHealth.isReplicatedOkAfterPending()) { request.getReplicationQueue().enqueue(overHealth); } return true; @@ -124,35 +113,6 @@ public boolean handle(ContainerCheckRequest request) { return false; } - /** - * Verify there's no under or over replication if only healthy replicas are - * considered, or that there are no healthy replicas. - * @return true if there's no under/over replication considering healthy - * replicas - */ - private boolean verifyPerfectReplication(ContainerCheckRequest request) { - RatisContainerReplicaCount replicaCountWithoutUnhealthy = - new RatisContainerReplicaCount(request.getContainerInfo(), - request.getContainerReplicas(), request.getPendingOps(), - request.getMaintenanceRedundancy(), false); - - if (replicaCountWithoutUnhealthy.getHealthyReplicaCount() == 0) { - return true; - } - if (replicaCountWithoutUnhealthy.isUnderReplicated() || - replicaCountWithoutUnhealthy.isOverReplicated()) { - LOG.debug("Checking replication for container {} without considering " + - "UNHEALTHY replicas. isUnderReplicated is [{}]. " + - "isOverReplicated is [{}]. Returning false because there should" + - " be perfect replication for this handler to work.", - request.getContainerInfo(), - replicaCountWithoutUnhealthy.isUnderReplicated(), - replicaCountWithoutUnhealthy.isOverReplicated()); - return false; - } - return true; - } - /** * Checks if the container is over or under replicated. */ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java index ecbaaea6ccb..a9e7d71b442 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java @@ -641,13 +641,11 @@ void testOverReplicatedWithAndWithoutPending() { * A container is safely over replicated if: * 1. It is over replicated. * 2. Has at least replication factor number of matching replicas. - * 3. # matching replicas - replication factor >= pending deletes. */ @Test void testSafelyOverReplicated() { /* First case: 3 CLOSED, 2 UNHEALTHY, 1 pending delete. - Expectation: Not safely over replicated because rule 3 is violated. */ ContainerInfo container = createContainerInfo(RatisReplicationConfig.getInstance( @@ -665,12 +663,13 @@ void testSafelyOverReplicated() { RatisContainerReplicaCount withoutUnhealthy = new RatisContainerReplicaCount(container, replicas, ops, 2, false); validate(withoutUnhealthy, true, 0, false, false); + // not safely over replicated (3 CLOSED - 1 pending delete) assertFalse(withoutUnhealthy.isSafelyOverReplicated()); RatisContainerReplicaCount withUnhealthy = new RatisContainerReplicaCount(container, replicas, ops, 2, true); validate(withUnhealthy, true, -1, true, false); - assertFalse(withUnhealthy.isSafelyOverReplicated()); + assertTrue(withUnhealthy.isSafelyOverReplicated()); /* Second case: 2 CLOSED, 1 CLOSING, 1 UNHEALTHY diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java index f69715f9b45..6f394f9213b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java @@ -217,6 +217,39 @@ public void testOverReplicatedQuasiClosedContainerWithDifferentOrigins() getOverReplicatedHealthResult(), 0); } + @Test + public void testClosedOverReplicatedWithAllUnhealthyReplicas() + throws IOException { + Set replicas = createReplicas(container.containerID(), + State.UNHEALTHY, 0, 0, 0, 0, 0); + List pendingOps = ImmutableList.of( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + MockDatanodeDetails.randomDatanodeDetails(), 0)); + + // 1 replica is already pending delete, so only 1 new command should be + // created + testProcessing(replicas, pendingOps, getOverReplicatedHealthResult(), + 1); + } + + @Test + public void testClosedOverReplicatedWithExcessUnhealthy() throws IOException { + Set replicas = createReplicas(container.containerID(), + State.CLOSED, 0, 0, 0); + ContainerReplica unhealthyReplica = + createContainerReplica(container.containerID(), 0, IN_SERVICE, + State.UNHEALTHY); + replicas.add(unhealthyReplica); + + Set>> commands = + testProcessing(replicas, Collections.emptyList(), + getOverReplicatedHealthResult(), + 1); + Pair> command = commands.iterator().next(); + assertEquals(unhealthyReplica.getDatanodeDetails(), + command.getKey()); + } + /** * Handler should not create any delete commands if removing a replica * makes the container mis replicated. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java index 5be1121263d..f31db93dfb2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java @@ -84,7 +84,6 @@ import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.createReplicasWithSameOrigin; import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.getNoNodesTestPlacementPolicy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -335,6 +334,8 @@ public void testUnderReplicatedClosedContainerWithOnlyUnhealthyReplicas( @Test public void testQuasiClosedContainerWithExcessUnhealthyReplica() throws IOException, NodeNotFoundException { + Mockito.when(nodeManager.getNodeStatus(any(DatanodeDetails.class))) + .thenReturn(NodeStatus.inServiceHealthy()); RatisReplicationConfig ratisRepConfig = RatisReplicationConfig.getInstance(THREE); ContainerInfo container = createContainerInfo(ratisRepConfig, 1, @@ -361,8 +362,6 @@ public void testQuasiClosedContainerWithExcessUnhealthyReplica() RatisOverReplicationHandler handler = new RatisOverReplicationHandler( ratisPlacementPolicy, replicationManager); - Mockito.when(nodeManager.getNodeStatus(any(DatanodeDetails.class))) - .thenReturn(NodeStatus.inServiceHealthy()); handler.processAndSendCommands(replicas, Collections.emptyList(), repQueue.dequeueOverReplicatedContainer(), 2); assertTrue(commandsSent.iterator().hasNext()); @@ -373,9 +372,55 @@ public void testQuasiClosedContainerWithExcessUnhealthyReplica() } + @Test + public void testClosedContainerWithOverReplicatedAllUnhealthy() + throws ContainerNotFoundException { + RatisReplicationConfig ratisRepConfig = + RatisReplicationConfig.getInstance(THREE); + ContainerInfo container = createContainerInfo(ratisRepConfig, 1, + HddsProtos.LifeCycleState.CLOSED); + Set replicas = + createReplicas(container.containerID(), + ContainerReplicaProto.State.UNHEALTHY, 0, 0, 0, 0); + storeContainerAndReplicas(container, replicas); + + replicationManager.processContainer(container, repQueue, repReport); + assertEquals(0, repReport.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + assertEquals(1, repReport.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(1, repQueue.overReplicatedQueueSize()); + } + + @Test + public void testClosedContainerWithExcessUnhealthy() + throws ContainerNotFoundException { + RatisReplicationConfig ratisRepConfig = + RatisReplicationConfig.getInstance(THREE); + ContainerInfo container = createContainerInfo(ratisRepConfig, 1, + HddsProtos.LifeCycleState.CLOSED); + Set replicas = + createReplicas(container.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0, 0); + ContainerReplica unhealthyReplica = + createContainerReplica(container.containerID(), 0, IN_SERVICE, + ContainerReplicaProto.State.UNHEALTHY); + replicas.add(unhealthyReplica); + storeContainerAndReplicas(container, replicas); + + replicationManager.processContainer(container, repQueue, repReport); + assertEquals(0, repReport.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + assertEquals(1, repReport.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(1, repQueue.overReplicatedQueueSize()); + } + @Test public void testQuasiClosedContainerWithUnhealthyReplicaOnUniqueOrigin() - throws IOException, NodeNotFoundException { + throws IOException { RatisReplicationConfig ratisRepConfig = RatisReplicationConfig.getInstance(THREE); ContainerInfo container = createContainerInfo(ratisRepConfig, 1, @@ -392,25 +437,10 @@ public void testQuasiClosedContainerWithUnhealthyReplicaOnUniqueOrigin() replicationManager.processContainer(container, repQueue, repReport); assertEquals(0, repReport.getStat( ReplicationManagerReport.HealthState.UNDER_REPLICATED)); - assertEquals(1, repReport.getStat( + assertEquals(0, repReport.getStat( ReplicationManagerReport.HealthState.OVER_REPLICATED)); assertEquals(0, repQueue.underReplicatedQueueSize()); - assertEquals(1, repQueue.overReplicatedQueueSize()); - - RatisOverReplicationHandler handler = new RatisOverReplicationHandler( - ratisPlacementPolicy, replicationManager); - - Mockito.when(nodeManager.getNodeStatus(any(DatanodeDetails.class))) - .thenReturn(NodeStatus.inServiceHealthy()); - handler.processAndSendCommands(replicas, Collections.emptyList(), - repQueue.dequeueOverReplicatedContainer(), 2); - assertTrue(commandsSent.iterator().hasNext()); - - // unhealthy replica can't be deleted because it has a unique origin DN - assertNotEquals(unhealthy.getDatanodeDetails().getUuid(), - commandsSent.iterator().next().getKey()); - assertEquals(SCMCommandProto.Type.deleteContainerCommand, - commandsSent.iterator().next().getValue().getType()); + assertEquals(0, repQueue.overReplicatedQueueSize()); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java index ad3f3fef6d7..f19c68a4b86 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -500,6 +501,43 @@ public void testOverReplicatedContainerWithMismatchedReplicas() { ReplicationManagerReport.HealthState.OVER_REPLICATED)); } + @Test + public void shouldQueueForOverReplicationOnlyWhenSafe() { + ContainerInfo container = + createContainerInfo(repConfig, 1L, HddsProtos.LifeCycleState.CLOSED); + Set replicas = createReplicas(container.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0); + ContainerReplica unhealthyReplica = + createContainerReplica(container.containerID(), 0, IN_SERVICE, + ContainerReplicaProto.State.UNHEALTHY); + ContainerReplica mismatchedReplica = + createContainerReplica(container.containerID(), 0, IN_SERVICE, + ContainerReplicaProto.State.QUASI_CLOSED); + replicas.add(mismatchedReplica); + replicas.add(unhealthyReplica); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + + ContainerHealthResult.OverReplicatedHealthResult + result = (ContainerHealthResult.OverReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + + assertEquals(ContainerHealthResult.HealthState.OVER_REPLICATED, + result.getHealthState()); + assertEquals(1, result.getExcessRedundancy()); + assertFalse(result.isReplicatedOkAfterPending()); + + // not safe for over replication because we don't have 3 matching replicas + assertFalse(result.isSafelyOverReplicated()); + + assertTrue(healthCheck.handle(requestBuilder.build())); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(0, repQueue.overReplicatedQueueSize()); + assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + /** * Scenario: CLOSED container with 2 CLOSED, 1 CLOSING and 3 UNHEALTHY * replicas. @@ -544,6 +582,44 @@ public void testHandlerReturnsTrueForExcessUnhealthyReplicas() { ReplicationManagerReport.HealthState.UNDER_REPLICATED)); } + /** + * There is a CLOSED container with 3 CLOSED replicas and 1 QUASI_CLOSED + * replica with incorrect sequence ID. This container is over replicated + * because of the QUASI_CLOSED replica and should be queued for over + * replication. + */ + @Test + public void testExcessQuasiClosedWithIncorrectSequenceID() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), State.CLOSED, 0, 0, 0); + ContainerReplica quasiClosed = + createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, container.getSequenceId() - 1); + replicas.add(quasiClosed); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + ContainerHealthResult result = + healthCheck.checkHealth(requestBuilder.build()); + + // there's an excess QUASI_CLOSED replica with incorrect sequence ID, so + // it's over replicated + assertEquals(HealthState.OVER_REPLICATED, result.getHealthState()); + OverReplicatedHealthResult overRepResult = + (OverReplicatedHealthResult) result; + assertEquals(1, overRepResult.getExcessRedundancy()); + assertFalse(overRepResult.hasMismatchedReplicas()); + assertFalse(overRepResult.isReplicatedOkAfterPending()); + + assertTrue(healthCheck.handle(requestBuilder.build())); + assertEquals(0, repQueue.underReplicatedQueueSize()); + assertEquals(1, repQueue.overReplicatedQueueSize()); + assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + /** * Scenario: CLOSED container with 3 CLOSED and 3 UNHEALTHY replicas. * Expectation: This container should be queued for over replication diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java index 91b2a143c19..07c99ca364f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java @@ -133,8 +133,12 @@ public void shouldReturnFalseForOverReplicatedHealthyReplicas() { assertFalse(handler.handle(requestBuilder.build())); } + /** + * Handling this is the responsibility of + * {@link RatisReplicationCheckHandler}, so this handler should return false. + */ @Test - public void shouldReturnTrueForExcessUnhealthyReplicas() { + public void shouldReturnFalseForExcessUnhealthyReplicas() { ContainerInfo container = createContainerInfo(repConfig, 1L, HddsProtos.LifeCycleState.CLOSED); Set replicas @@ -147,20 +151,12 @@ public void shouldReturnTrueForExcessUnhealthyReplicas() { requestBuilder.setContainerReplicas(replicas) .setContainerInfo(container); - ContainerHealthResult.OverReplicatedHealthResult result = - (ContainerHealthResult.OverReplicatedHealthResult) - handler.checkReplication(requestBuilder.build()); - assertEquals(ContainerHealthResult.HealthState.OVER_REPLICATED, - result.getHealthState()); - assertEquals(1, result.getExcessRedundancy()); - assertFalse(result.isReplicatedOkAfterPending()); - - assertTrue(handler.handle(requestBuilder.build())); + assertFalse(handler.handle(requestBuilder.build())); assertEquals(0, repQueue.underReplicatedQueueSize()); - assertEquals(1, repQueue.overReplicatedQueueSize()); - assertEquals(1, report.getStat( + assertEquals(0, repQueue.overReplicatedQueueSize()); + assertEquals(0, report.getStat( ReplicationManagerReport.HealthState.OVER_REPLICATED)); - assertEquals(1, + assertEquals(0, report.getStat(ReplicationManagerReport.HealthState.UNHEALTHY)); } @@ -266,19 +262,15 @@ public void testUnderReplicatedDueToPendingDelete() { } @Test - public void testOverReplicationFixedByPendingDelete() { + public void testOverReplicationWithAllUnhealthyReplicas() { ContainerInfo container = createContainerInfo(repConfig, 1L, HddsProtos.LifeCycleState.CLOSED); Set replicas = createReplicas(container.containerID(), ContainerReplicaProto.State.UNHEALTHY, 0, 0, 0, 0); - List pendingOps = - ImmutableList.of(ContainerReplicaOp.create( - ContainerReplicaOp.PendingOpType.DELETE, - replicas.stream().findFirst().get().getDatanodeDetails(), 0)); - requestBuilder.setContainerReplicas(replicas) - .setContainerInfo(container) - .setPendingOps(pendingOps); + + requestBuilder.setContainerInfo(container) + .setContainerReplicas(replicas); ContainerHealthResult.OverReplicatedHealthResult result = (ContainerHealthResult.OverReplicatedHealthResult) @@ -287,11 +279,11 @@ public void testOverReplicationFixedByPendingDelete() { assertEquals(ContainerHealthResult.HealthState.OVER_REPLICATED, result.getHealthState()); assertEquals(1, result.getExcessRedundancy()); - assertTrue(result.isReplicatedOkAfterPending()); + assertFalse(result.isReplicatedOkAfterPending()); assertTrue(handler.handle(requestBuilder.build())); assertEquals(0, repQueue.underReplicatedQueueSize()); - assertEquals(0, repQueue.overReplicatedQueueSize()); + assertEquals(1, repQueue.overReplicatedQueueSize()); assertEquals(1, report.getStat( ReplicationManagerReport.HealthState.OVER_REPLICATED)); assertEquals(1, @@ -299,22 +291,19 @@ public void testOverReplicationFixedByPendingDelete() { } @Test - public void shouldQueueForOverReplicationOnlyWhenSafe() { + public void testOverReplicationFixedByPendingDelete() { ContainerInfo container = createContainerInfo(repConfig, 1L, HddsProtos.LifeCycleState.CLOSED); - Set replicas = createReplicas(container.containerID(), - ContainerReplicaProto.State.CLOSED, 0, 0); - ContainerReplica unhealthyReplica = - createContainerReplica(container.containerID(), 0, IN_SERVICE, - ContainerReplicaProto.State.UNHEALTHY); - ContainerReplica mismatchedReplica = - createContainerReplica(container.containerID(), 0, IN_SERVICE, - ContainerReplicaProto.State.QUASI_CLOSED); - replicas.add(mismatchedReplica); - replicas.add(unhealthyReplica); - + Set replicas + = createReplicas(container.containerID(), + ContainerReplicaProto.State.UNHEALTHY, 0, 0, 0, 0); + List pendingOps = + ImmutableList.of(ContainerReplicaOp.create( + ContainerReplicaOp.PendingOpType.DELETE, + replicas.stream().findFirst().get().getDatanodeDetails(), 0)); requestBuilder.setContainerReplicas(replicas) - .setContainerInfo(container); + .setContainerInfo(container) + .setPendingOps(pendingOps); ContainerHealthResult.OverReplicatedHealthResult result = (ContainerHealthResult.OverReplicatedHealthResult) @@ -323,10 +312,7 @@ public void shouldQueueForOverReplicationOnlyWhenSafe() { assertEquals(ContainerHealthResult.HealthState.OVER_REPLICATED, result.getHealthState()); assertEquals(1, result.getExcessRedundancy()); - assertFalse(result.isReplicatedOkAfterPending()); - - // not safe for over replication because we don't have 3 matching replicas - assertFalse(result.isSafelyOverReplicated()); + assertTrue(result.isReplicatedOkAfterPending()); assertTrue(handler.handle(requestBuilder.build())); assertEquals(0, repQueue.underReplicatedQueueSize()); @@ -366,18 +352,11 @@ public void overReplicationCheckWithQuasiClosedReplica() { final ContainerInfo container = ReplicationTestUtil.createContainerInfo( repConfig, 1, HddsProtos.LifeCycleState.CLOSED, sequenceID); - final Set replicas = new HashSet<>(2); - replicas.add(createContainerReplica(container.containerID(), 0, - IN_SERVICE, State.CLOSED, sequenceID)); - replicas.add(createContainerReplica(container.containerID(), 0, - IN_SERVICE, State.CLOSED, sequenceID)); - replicas.add(createContainerReplica(container.containerID(), 0, - IN_SERVICE, State.CLOSED, sequenceID)); - - final ContainerReplica quasiClosedReplica = - createContainerReplica(container.containerID(), 0, - IN_SERVICE, State.QUASI_CLOSED, sequenceID - 1); - replicas.add(quasiClosedReplica); + Set replicas = new HashSet<>(4); + for (int i = 0; i < 4; i++) { + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, sequenceID - 1)); + } requestBuilder.setContainerReplicas(replicas).setContainerInfo(container); assertTrue(handler.handle(requestBuilder.build())); @@ -385,10 +364,33 @@ public void overReplicationCheckWithQuasiClosedReplica() { assertEquals(1, repQueue.overReplicatedQueueSize()); assertEquals(1, report.getStat( ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertEquals(0, + report.getStat(ReplicationManagerReport.HealthState.UNDER_REPLICATED)); assertEquals(1, report.getStat(ReplicationManagerReport.HealthState.UNHEALTHY)); } + @Test + public void testUnderReplicatedWithQuasiClosedReplicasHavingLessSequenceId() { + final long sequenceID = 20; + final ContainerInfo container = ReplicationTestUtil.createContainerInfo( + repConfig, 1, HddsProtos.LifeCycleState.CLOSED, sequenceID); + + Set replicas = new HashSet<>(2); + for (int i = 0; i < 2; i++) { + replicas.add(createContainerReplica(container.containerID(), 0, + IN_SERVICE, State.QUASI_CLOSED, sequenceID - 1)); + } + + requestBuilder.setContainerReplicas(replicas).setContainerInfo(container); + assertTrue(handler.handle(requestBuilder.build())); + assertEquals(1, repQueue.underReplicatedQueueSize()); + assertEquals(0, repQueue.overReplicatedQueueSize()); + assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertEquals(1, + report.getStat(ReplicationManagerReport.HealthState.UNHEALTHY)); + } } From cd9bcc6abb59b6080fdc5e32313f61b0ab65b3b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 18:41:41 +0100 Subject: [PATCH 38/51] HDDS-9812. Bump gradle-enterprise-maven-extension to 1.19.3 (#5680) --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 6fb27c8aed3..de4bd7de54d 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -24,7 +24,7 @@ com.gradle gradle-enterprise-maven-extension - 1.19.2 + 1.19.3 com.gradle From 40a8e6729cc6da9bd3362d1d7017ab44436c5b1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Dec 2023 07:46:21 +0100 Subject: [PATCH 39/51] HDDS-9813. Bump native-maven-plugin to 1.0-alpha-11 (#5684) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9cfbee8d2d0..dbeda25b411 100644 --- a/pom.xml +++ b/pom.xml @@ -283,7 +283,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.3.1 2.3.0 1.0-beta-1 - 1.0-alpha-8 + 1.0-alpha-11 3.1.2 3.9.1 3.1.0 From 74dc25b1ccc43add843b62e57c1cc13eb11e6431 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Fri, 1 Dec 2023 17:06:00 +0530 Subject: [PATCH 40/51] HDDS-9530. Recon - NPE in handling deleteKey event in NSSummaryFSO task. (#5490) --- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 47 ++++++++++++------- .../recon/tasks/TestOMDBUpdatesHandler.java | 19 ++------ 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java index 944ae322443..cfaf4bb60a8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java @@ -81,12 +81,13 @@ public void delete(int cfIndex, byte[] keyBytes) { } /** + * Processes an OM DB update event based on the provided parameters. * - * @param cfIndex - * @param keyBytes - * @param valueBytes - * @param action - * @throws IOException + * @param cfIndex Index of the column family. + * @param keyBytes Serialized key bytes. + * @param valueBytes Serialized value bytes. + * @param action Type of the database action (e.g., PUT, DELETE). + * @throws IOException If an I/O error occurs. */ private void processEvent(int cfIndex, byte[] keyBytes, byte[] valueBytes, OMDBUpdateEvent.OMDBUpdateAction action) @@ -111,10 +112,12 @@ private void processEvent(int cfIndex, byte[] keyBytes, byte[] final Object key = cf.getKeyCodec().fromPersistedFormat(keyBytes); builder.setKey(key); - // Put new - // Put existing --> Update - // Delete existing - // Delete non-existing + // Handle the event based on its type: + // - PUT with a new key: Insert the new value. + // - PUT with an existing key: Update the existing value. + // - DELETE with an existing key: Remove the value. + // - DELETE with a non-existing key: No action, log a warning if + // necessary. Table table = omMetadataManager.getTable(tableName); OMDBUpdateEvent latestEvent = omdbLatestUpdateEvents.get(key); @@ -127,7 +130,7 @@ private void processEvent(int cfIndex, byte[] keyBytes, byte[] oldValue = table.getSkipCache(key); } - if (action == PUT) { + if (action.equals(PUT)) { final Object value = cf.getValueCodec().fromPersistedFormat(valueBytes); // If the updated value is not valid for this event, we skip it. @@ -137,8 +140,7 @@ private void processEvent(int cfIndex, byte[] keyBytes, byte[] } builder.setValue(value); - // If a PUT operation happens on an existing Key, it is tagged - // as an "UPDATE" event. + // Tag PUT operations on existing keys as "UPDATE" events. if (oldValue != null) { // If the oldValue is not valid for this event, we skip it. @@ -153,8 +155,22 @@ private void processEvent(int cfIndex, byte[] keyBytes, byte[] } } } else if (action.equals(DELETE)) { - if (oldValue != null && !omUpdateEventValidator.isValidEvent(tableName, - oldValue, key, action)) { + if (null == oldValue) { + String keyStr = (key instanceof String) ? key.toString() : ""; + if (keyStr.isEmpty()) { + LOG.warn( + "Only DTOKEN_TABLE table uses OzoneTokenIdentifier as key " + + "instead of String. Event on any other table in this " + + "condition may need to be investigated. This DELETE " + + "event is on {} table which is not useful for Recon to " + + "capture.", tableName); + } + LOG.warn("Old Value of Key: {} in table: {} should not be null " + + "for DELETE event ", keyStr, tableName); + return; + } + if (!omUpdateEventValidator.isValidEvent(tableName, oldValue, key, + action)) { return; } // When you delete a Key, we add the old value to the event so that @@ -170,8 +186,7 @@ private void processEvent(int cfIndex, byte[] keyBytes, byte[] omdbUpdateEvents.add(event); omdbLatestUpdateEvents.put(key, event); } else { - // key type or value type cannot be determined for this table. - // log a warn message and ignore the update. + // Log and ignore events if key or value types are undetermined. if (LOG.isWarnEnabled()) { LOG.warn(String.format("KeyType or ValueType could not be determined" + " for table %s. Ignoring the event.", tableName)); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java index 9ece45df95e..dca0da341a2 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java @@ -150,8 +150,6 @@ public void testPut() throws Exception { public void testDelete() throws Exception { // Write 1 volume, 1 key into source and target OM DBs. String volumeKey = omMetadataManager.getVolumeKey("sampleVol"); - String nonExistVolumeKey = omMetadataManager - .getVolumeKey("nonExistingVolume"); OmVolumeArgs args = OmVolumeArgs.newBuilder() .setVolume("sampleVol") @@ -181,7 +179,9 @@ public void testDelete() throws Exception { OMDBUpdatesHandler omdbUpdatesHandler = captureEvents(writeBatches); List events = omdbUpdatesHandler.getEvents(); - assertEquals(4, events.size()); + + // Assert for non existent keys, no events will be captured and handled. + assertEquals(2, events.size()); OMDBUpdateEvent keyEvent = events.get(0); assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, keyEvent.getAction()); @@ -194,19 +194,6 @@ public void testDelete() throws Exception { assertNotNull(volEvent.getValue()); OmVolumeArgs volumeInfo = (OmVolumeArgs) volEvent.getValue(); assertEquals("sampleVol", volumeInfo.getVolume()); - - // Assert the values of non existent keys are set to null. - OMDBUpdateEvent nonExistKey = events.get(2); - assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, - nonExistKey.getAction()); - assertEquals("/sampleVol/bucketOne/key_two", nonExistKey.getKey()); - assertNull(nonExistKey.getValue()); - - OMDBUpdateEvent nonExistVolume = events.get(3); - assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, - nonExistVolume.getAction()); - assertEquals(nonExistVolumeKey, nonExistVolume.getKey()); - assertNull(nonExistVolume.getValue()); } @Test From 5a9dbb85032cb3c5c6066010c8e9e91c680d6301 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Fri, 1 Dec 2023 11:55:06 +0000 Subject: [PATCH 41/51] HDDS-9729. Provide API to check a container via Replication Manager (#5632) --- .../replication/ContainerCheckRequest.java | 13 +++++- .../replication/NullReplicationQueue.java | 40 +++++++++++++++++++ .../replication/ReplicationManager.java | 29 ++++++++++++++ .../ClosedWithUnhealthyReplicasHandler.java | 6 ++- .../health/ClosingContainerHandler.java | 7 ++++ .../health/DeletingContainerHandler.java | 6 +++ .../health/ECReplicationCheckHandler.java | 2 +- .../health/EmptyContainerHandler.java | 16 ++++---- .../health/MismatchedReplicasHandler.java | 3 ++ .../health/OpenContainerHandler.java | 5 ++- .../health/QuasiClosedContainerHandler.java | 10 ++++- .../replication/TestReplicationManager.java | 25 ++++++++++++ ...estClosedWithUnhealthyReplicasHandler.java | 13 ++++++ .../health/TestClosingContainerHandler.java | 10 +++-- .../health/TestDeletingContainerHandler.java | 20 +++++++--- .../health/TestEmptyContainerHandler.java | 18 +++++++++ .../health/TestMismatchedReplicasHandler.java | 33 +++++++++++++++ .../health/TestOpenContainerHandler.java | 16 ++++++++ .../TestQuasiClosedContainerHandler.java | 20 +++++++++- 19 files changed, 267 insertions(+), 25 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/NullReplicationQueue.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java index 9d51e20c33b..93035b696cd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java @@ -36,7 +36,7 @@ public final class ContainerCheckRequest { private final int maintenanceRedundancy; private final ReplicationManagerReport report; private final ReplicationQueue replicationQueue; - + private final boolean readOnly; private ContainerCheckRequest(Builder builder) { this.containerInfo = builder.containerInfo; @@ -46,6 +46,7 @@ private ContainerCheckRequest(Builder builder) { this.maintenanceRedundancy = builder.maintenanceRedundancy; this.report = builder.report; this.replicationQueue = builder.replicationQueue; + this.readOnly = builder.readOnly; } public List getPendingOps() { @@ -72,6 +73,10 @@ public ReplicationQueue getReplicationQueue() { return replicationQueue; } + public boolean isReadOnly() { + return readOnly; + } + /** * Builder class for ContainerCheckRequest. */ @@ -83,6 +88,7 @@ public static class Builder { private int maintenanceRedundancy; private ReplicationManagerReport report; private ReplicationQueue replicationQueue; + private boolean readOnly = false; public Builder setContainerInfo(ContainerInfo containerInfo) { this.containerInfo = containerInfo; @@ -115,6 +121,11 @@ public Builder setReport(ReplicationManagerReport report) { return this; } + public Builder setReadOnly(boolean readOnly) { + this.readOnly = readOnly; + return this; + } + public ContainerCheckRequest build() { return new ContainerCheckRequest(this); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/NullReplicationQueue.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/NullReplicationQueue.java new file mode 100644 index 00000000000..4e6148830eb --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/NullReplicationQueue.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.container.replication; + +/** + * A class which extents ReplicationQueue and does nothing. This is used when + * checking containers in a read-only mode, where we don't want to queue them + * for replication. + */ + +public class NullReplicationQueue extends ReplicationQueue { + + @Override + public void enqueue(ContainerHealthResult.UnderReplicatedHealthResult + underReplicatedHealthResult) { + // Do nothing + } + + @Override + public void enqueue(ContainerHealthResult.OverReplicatedHealthResult + overReplicatedHealthResult) { + // Do nothing + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java index d34d14ad210..3b9f66595f4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java @@ -197,6 +197,8 @@ public class ReplicationManager implements SCMService { private final UnderReplicatedProcessor underReplicatedProcessor; private final OverReplicatedProcessor overReplicatedProcessor; private final HealthCheck containerCheckChain; + private final ReplicationQueue nullReplicationQueue = + new NullReplicationQueue(); /** * Constructs ReplicationManager instance with the given configuration. @@ -840,6 +842,12 @@ private void addExcludedNode(DatanodeDetails dn) { protected void processContainer(ContainerInfo containerInfo, ReplicationQueue repQueue, ReplicationManagerReport report) throws ContainerNotFoundException { + processContainer(containerInfo, repQueue, report, false); + } + + protected boolean processContainer(ContainerInfo containerInfo, + ReplicationQueue repQueue, ReplicationManagerReport report, + boolean readOnly) throws ContainerNotFoundException { synchronized (containerInfo) { ContainerID containerID = containerInfo.containerID(); final boolean isEC = isEC(containerInfo.getReplicationConfig()); @@ -856,6 +864,7 @@ protected void processContainer(ContainerInfo containerInfo, .setReport(report) .setPendingOps(pendingOps) .setReplicationQueue(repQueue) + .setReadOnly(readOnly) .build(); // This will call the chain of container health handlers in turn which // will issue commands as needed, update the report and perhaps add @@ -865,6 +874,7 @@ protected void processContainer(ContainerInfo containerInfo, LOG.debug("Container {} had no actions after passing through the " + "check chain", containerInfo.containerID()); } + return handled; } } @@ -969,6 +979,25 @@ public ContainerHealthResult getContainerReplicationHealth( } } + /** + * This method is used to check the container health status. It runs all the + * same checks ReplicationManager runs against a container to determine if it + * is under replicated or over replicated etc, but in a readOnly mode so no + * commands are sent. The passed in ReplicationManagerReport is updated and + * the caller can query it on return to see the results of the check. + * @param containerInfo The container to check + * @param report The instance of the replicationManager report to update with + * the results of the check. + * @return True if the handler chain took action on the request or false other + * wise. If the method returns false, then the container is deemed + * healthy by replication manager. + */ + public boolean checkContainerStatus(ContainerInfo containerInfo, + ReplicationManagerReport report) throws ContainerNotFoundException { + report.increment(containerInfo.getState()); + return processContainer(containerInfo, nullReplicationQueue, report, true); + } + /** * Retrieve a list of any pending container replications or deletes for the * given containerID. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java index b5dd9ed3c38..1b23c7580b6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java @@ -102,11 +102,13 @@ public boolean handle(ContainerCheckRequest request) { } foundUnhealthy = true; - sendDeleteCommand(containerInfo, replica); + if (!request.isReadOnly()) { + sendDeleteCommand(containerInfo, replica); + } } } - // some unhealthy replicas were found so the container must be over + // some unhealthy replicas were found so the container must be // over replicated due to unhealthy replicas. if (foundUnhealthy) { request.getReport().incrementAndSample( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java index 428d12e28e5..97beb1dce20 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java @@ -70,12 +70,19 @@ public boolean handle(ContainerCheckRequest request) { boolean forceClose = containerInfo.getReplicationConfig() .getReplicationType() != ReplicationType.RATIS; + // TODO - review this logic - may need an empty check here if (request.getContainerReplicas().isEmpty()) { request.getReport().incrementAndSample( ReplicationManagerReport.HealthState.MISSING, containerInfo.containerID()); } + if (request.isReadOnly()) { + // The reset of this method modifies container state, so we just return + // here if the request is read only. + return true; + } + boolean allUnhealthy = true; for (ContainerReplica replica : request.getContainerReplicas()) { if (replica.getState() != ContainerReplicaProto.State.UNHEALTHY) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java index d4b8b262b5c..eaaee485461 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java @@ -70,6 +70,12 @@ public boolean handle(ContainerCheckRequest request) { LOG.debug("Checking container {} in DeletingContainerHandler", containerInfo); + if (request.isReadOnly()) { + // The reset of this method modifies state, so we just return true if + // the request is read only + return true; + } + if (request.getContainerReplicas().size() == 0) { LOG.debug("Deleting Container {} has no replicas so marking for cleanup" + " and returning true", containerInfo); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java index f4677f82151..197b53ed900 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java @@ -131,7 +131,7 @@ public ContainerHealthResult checkHealth(ContainerCheckRequest request) { // via an EC reconstruction command. Note that it may also have some // replicas in decommission / maintenance states, but as the under // replication is not caused only by decommission, we say it is not - // due to decommission/ + // due to decommission dueToOutOfService = false; remainingRedundancy = repConfig.getParity() - missingIndexes.size(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java index 69804698f29..19f5097a274 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java @@ -60,14 +60,16 @@ public boolean handle(ContainerCheckRequest request) { request.getReport() .incrementAndSample(ReplicationManagerReport.HealthState.EMPTY, containerInfo.containerID()); - LOG.debug("Container {} is empty and closed, marking as DELETING", - containerInfo); - // delete replicas if they are closed and empty - deleteContainerReplicas(containerInfo, replicas); + if (!request.isReadOnly()) { + LOG.debug("Container {} is empty and closed, marking as DELETING", + containerInfo); + // delete replicas if they are closed and empty + deleteContainerReplicas(containerInfo, replicas); - // Update the container's state - replicationManager.updateContainerState( - containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE); + // Update the container's state + replicationManager.updateContainerState( + containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE); + } return true; } else if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED && containerInfo.getNumberOfKeys() == 0 && replicas.isEmpty()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java index ecfdf0c90e7..68df0e7ed48 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java @@ -68,6 +68,9 @@ public boolean handle(ContainerCheckRequest request) { LOG.debug("Checking container {} in MismatchedReplicasHandler", containerInfo); + if (request.isReadOnly()) { + return false; + } // close replica if needed for (ContainerReplica replica : replicas) { if (shouldBeClosed(containerInfo, replica)) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java index a644f5e8341..2c0b405db97 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java @@ -62,7 +62,10 @@ public boolean handle(ContainerCheckRequest request) { request.getReport().incrementAndSample( ReplicationManagerReport.HealthState.OPEN_UNHEALTHY, containerInfo.containerID()); - replicationManager.sendCloseContainerEvent(containerInfo.containerID()); + if (!request.isReadOnly()) { + replicationManager + .sendCloseContainerEvent(containerInfo.containerID()); + } return true; } // For open containers we do not want to do any further processing in RM diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java index 01f6a05d77d..94a93af1074 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java @@ -36,6 +36,9 @@ * Class for handling containers that are in QUASI_CLOSED state. This will * send commands to Datanodes to force close these containers if they satisfy * the requirements to be force closed. Only meant for RATIS containers. + * + * Note - this handler always returns false so further handlers can can for + * under and over replication etc. */ public class QuasiClosedContainerHandler extends AbstractCheck { public static final Logger LOG = @@ -69,8 +72,9 @@ public boolean handle(ContainerCheckRequest request) { Set replicas = request.getContainerReplicas(); if (canForceCloseContainer(containerInfo, replicas)) { - forceCloseContainer(containerInfo, replicas); - return true; + if (!request.isReadOnly()) { + forceCloseContainer(containerInfo, replicas); + } } else { LOG.debug("Container {} cannot be force closed and is stuck in " + "QUASI_CLOSED", containerInfo); @@ -78,6 +82,8 @@ public boolean handle(ContainerCheckRequest request) { ReplicationManagerReport.HealthState.QUASI_CLOSED_STUCK, containerInfo.containerID()); } + // Always return false, even if commands were sent. That way, under and + // over replication handlers can to check for other issues in the container. return false; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java index f31db93dfb2..d2b2d18d358 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java @@ -510,6 +510,17 @@ public void testHealthyContainer() throws ContainerNotFoundException { assertEquals(0, repQueue.overReplicatedQueueSize()); } + @Test + public void testHealthyContainerStatus() throws ContainerNotFoundException { + ContainerInfo container = createContainerInfo(repConfig, 1, + HddsProtos.LifeCycleState.CLOSED); + addReplicas(container, ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4, 5); + + boolean result = replicationManager.checkContainerStatus( + container, repReport); + assertEquals(false, result); + } + @Test public void testUnderReplicatedContainer() throws ContainerNotFoundException { ContainerInfo container = createContainerInfo(repConfig, 1, @@ -524,6 +535,20 @@ public void testUnderReplicatedContainer() throws ContainerNotFoundException { ReplicationManagerReport.HealthState.UNDER_REPLICATED)); } + @Test + public void testUnderReplicatedContainerStatus() + throws ContainerNotFoundException { + ContainerInfo container = createContainerInfo(repConfig, 1, + HddsProtos.LifeCycleState.CLOSED); + addReplicas(container, ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4); + + boolean result = replicationManager.checkContainerStatus( + container, repReport); + assertEquals(1, repReport.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + assertEquals(true, result); + } + /** * {@link * ReplicationManager#getContainerReplicationHealth(ContainerInfo, Set)} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java index 073717d26ef..921538da949 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java @@ -137,10 +137,23 @@ public void testRatisContainerReturnsFalse() { .setContainerInfo(container) .build(); + ContainerCheckRequest readRequest = requestBuilder + .setContainerReplicas(containerReplicas) + .setContainerInfo(container) + .setReadOnly(true) + .build(); + assertTrue(handler.handle(request)); assertEquals(1, request.getReport().getStat( ReplicationManagerReport.HealthState.OVER_REPLICATED)); + assertTrue(handler.handle(readRequest)); + // Same report object is incremented again + assertEquals(2, request.getReport().getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + + // Only a single delete should be sent, as the read request should not have + // triggered one. ArgumentCaptor replicaIndexCaptor = ArgumentCaptor.forClass(Integer.class); Mockito.verify(replicationManager, Mockito.times(2)) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java index a89730424a1..c04b8b2fc7a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java @@ -191,13 +191,17 @@ public void testClosingContainerStateIsNotUpdatedWhenThereAreReplicas() { ReplicationManagerReport report = new ReplicationManagerReport(); - ContainerCheckRequest request = new ContainerCheckRequest.Builder() + ContainerCheckRequest.Builder builder = new ContainerCheckRequest.Builder() .setPendingOps(Collections.emptyList()) .setReport(report) .setContainerInfo(containerInfo) - .setContainerReplicas(containerReplicas) - .build(); + .setContainerReplicas(containerReplicas); + ContainerCheckRequest request = builder.build(); + + builder.setReadOnly(true); + ContainerCheckRequest readRequest = builder.build(); + assertAndVerify(readRequest, true, 0); assertAndVerify(request, true, 3); report.getStats().forEach((k, v) -> Assertions.assertEquals(0L, v)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java index bc233754f1a..fc63a825dc2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java @@ -126,24 +126,32 @@ public void testCleanupIfNoReplicaExist() { HddsProtos.ReplicationFactor.THREE), 1); //ec container - //since updateContainerState is called once when testing - //ratis container, so here should be 1+1 = 2 times - cleanupIfNoReplicaExist(ecReplicationConfig, 2); + cleanupIfNoReplicaExist(ecReplicationConfig, 1); } private void cleanupIfNoReplicaExist( ReplicationConfig replicationConfig, int times) { + Mockito.clearInvocations(replicationManager); ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( replicationConfig, 1, DELETING); Set containerReplicas = new HashSet<>(); - ContainerCheckRequest request = new ContainerCheckRequest.Builder() + ContainerCheckRequest.Builder builder = new ContainerCheckRequest.Builder() .setPendingOps(Collections.emptyList()) .setReport(new ReplicationManagerReport()) .setContainerInfo(containerInfo) - .setContainerReplicas(containerReplicas) - .build(); + .setContainerReplicas(containerReplicas); + + ContainerCheckRequest request = builder.build(); + + builder.setReadOnly(true); + ContainerCheckRequest readRequest = builder.build(); + + Assertions.assertTrue(deletingContainerHandler.handle(readRequest)); + Mockito.verify(replicationManager, Mockito.times(0)) + .updateContainerState(Mockito.any(ContainerID.class), + Mockito.any(HddsProtos.LifeCycleEvent.class)); Assertions.assertTrue(deletingContainerHandler.handle(request)); Mockito.verify(replicationManager, Mockito.times(times)) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java index 760ec9b7d45..e89164250a5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java @@ -87,6 +87,15 @@ public void testEmptyAndClosedECContainerReturnsTrue() .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); + + assertAndVerify(readRequest, true, 0, 1); assertAndVerify(request, true, 5, 1); } @@ -109,6 +118,15 @@ public void testEmptyAndClosedRatisContainerReturnsTrue() .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); + + assertAndVerify(readRequest, true, 0, 1); assertAndVerify(request, true, 3, 1); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java index cb5f4649e46..db875bc6884 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java @@ -128,9 +128,18 @@ public void testCloseCommandSentForMismatchedReplicas() { .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); + // this handler always returns false so other handlers can fix issues // such as under replication Assertions.assertFalse(handler.handle(request)); + Assertions.assertFalse(handler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerReplicaCommand( @@ -207,10 +216,18 @@ public void testCloseCommandSentForMismatchedRatisReplicas() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); // this handler always returns false so other handlers can fix issues // such as under replication Assertions.assertFalse(handler.handle(request)); + Assertions.assertFalse(handler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerReplicaCommand( @@ -254,10 +271,18 @@ public void testCloseCommandSentForMismatchedReplicaOfQuasiClosedContainer() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); // this handler always returns false so other handlers can fix issues // such as under replication Assertions.assertFalse(handler.handle(request)); + Assertions.assertFalse(handler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerReplicaCommand( @@ -296,10 +321,18 @@ public void testQuasiClosedReplicaOfClosedContainer() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); // this handler always returns false so other handlers can fix issues // such as under replication Assertions.assertFalse(handler.handle(request)); + Assertions.assertFalse(handler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerReplicaCommand( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java index 4ed8beea750..32ef98e76a5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java @@ -107,7 +107,15 @@ public void testOpenUnhealthyContainerIsClosed() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); Assertions.assertTrue(openContainerHandler.handle(request)); + Assertions.assertTrue(openContainerHandler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerEvent(containerInfo.containerID()); } @@ -161,7 +169,15 @@ public void testOpenUnhealthyRatisContainerIsClosed() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); Assertions.assertTrue(openContainerHandler.handle(request)); + Assertions.assertTrue(openContainerHandler.handle(readRequest)); Mockito.verify(replicationManager, times(1)) .sendCloseContainerEvent(Mockito.any()); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java index da1b818a066..899b6571558 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java @@ -128,8 +128,16 @@ public void testQuasiClosedWithQuorumReturnsTrue() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); - Assertions.assertTrue(quasiClosedContainerHandler.handle(request)); + Assertions.assertFalse(quasiClosedContainerHandler.handle(request)); + Assertions.assertFalse(quasiClosedContainerHandler.handle(readRequest)); Mockito.verify(replicationManager, times(2)) .sendCloseContainerReplicaCommand(any(), any(), anyBoolean()); } @@ -224,8 +232,16 @@ public void testReplicasWithHighestBCSIDAreClosed() { .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); + ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.emptyList()) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .setReadOnly(true) + .build(); - Assertions.assertTrue(quasiClosedContainerHandler.handle(request)); + Assertions.assertFalse(quasiClosedContainerHandler.handle(request)); + Assertions.assertFalse(quasiClosedContainerHandler.handle(readRequest)); // verify close command was sent for replicas with sequence ID 1001, that // is dnTwo and dnThree Mockito.verify(replicationManager, times(1)) From a710a5f8fe912d639a981e8e41d8a5b8a024669f Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Fri, 1 Dec 2023 19:03:32 +0530 Subject: [PATCH 42/51] HDDS-9811. Follower SCM should not process Pipeline Action (#5712) --- .../scm/pipeline/PipelineActionHandler.java | 60 ++++++++---- .../pipeline/TestPipelineActionHandler.java | 95 +++++++++++++++---- 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java index 2f1785cf174..db6d6e77e07 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode; @@ -78,33 +79,56 @@ private void processPipelineAction(final DatanodeDetails datanode, final ClosePipelineInfo info = pipelineAction.getClosePipeline(); final PipelineAction.Action action = pipelineAction.getAction(); final PipelineID pid = PipelineID.getFromProtobuf(info.getPipelineID()); - try { - LOG.info("Received pipeline action {} for {} from datanode {}. " + - "Reason : {}", action, pid, datanode.getUuidString(), - info.getDetailedReason()); + final String logMsg = "Received pipeline action " + action + " for " + pid + + " from datanode " + datanode.getUuidString() + "." + + " Reason : " + info.getDetailedReason(); + + // We can skip processing Pipeline Action if the current SCM is not leader. + if (!scmContext.isLeader()) { + LOG.debug(logMsg); + LOG.debug("Cannot process Pipeline Action for pipeline {} as " + + "current SCM is not leader.", pid); + return; + } + + LOG.info(logMsg); + try { if (action == PipelineAction.Action.CLOSE) { pipelineManager.closePipeline(pid); } else { - LOG.error("unknown pipeline action:{}", action); + LOG.error("Received unknown pipeline action {}, for pipeline {} ", + action, pid); } } catch (PipelineNotFoundException e) { - LOG.warn("Pipeline action {} received for unknown pipeline {}, " + - "firing close pipeline event.", action, pid); - SCMCommand command = new ClosePipelineCommand(pid); - try { - command.setTerm(scmContext.getTermOfLeader()); - } catch (NotLeaderException nle) { - LOG.warn("Skip sending ClosePipelineCommand for pipeline {}," + - " since not leader SCM.", pid); - return; + closeUnknownPipeline(publisher, datanode, pid); + } catch (SCMException e) { + if (e.getResult() == SCMException.ResultCodes.SCM_NOT_LEADER) { + LOG.info("Cannot process Pipeline Action for pipeline {} as " + + "current SCM is not leader anymore.", pid); + } else { + LOG.error("Exception while processing Pipeline Action for Pipeline {}", + pid, e); } + } catch (IOException e) { + LOG.error("Exception while processing Pipeline Action for Pipeline {}", + pid, e); + } + } + + private void closeUnknownPipeline(final EventPublisher publisher, + final DatanodeDetails datanode, + final PipelineID pid) { + try { + LOG.warn("Pipeline action received for unknown Pipeline {}, " + + "firing close pipeline event.", pid); + SCMCommand command = new ClosePipelineCommand(pid); + command.setTerm(scmContext.getTermOfLeader()); publisher.fireEvent(SCMEvents.DATANODE_COMMAND, new CommandForDatanode<>(datanode.getUuid(), command)); - } catch (IOException ioe) { - LOG.error("Could not execute pipeline action={} pipeline={}", - action, pid, ioe); + } catch (NotLeaderException nle) { + LOG.info("Cannot process Pipeline Action for pipeline {} as " + + "current SCM is not leader anymore.", pid); } } - } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java index 791220f6707..0546d876171 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java @@ -18,10 +18,11 @@ package org.apache.hadoop.hdds.scm.pipeline; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineActionsProto; +import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; @@ -30,42 +31,98 @@ import org.mockito.Mockito; import java.io.IOException; -import java.util.UUID; /** * Test-cases to verify the functionality of PipelineActionHandler. */ public class TestPipelineActionHandler { + + @Test + public void testPipelineActionHandlerForValidPipeline() throws IOException { + + final PipelineManager manager = Mockito.mock(PipelineManager.class); + final EventQueue queue = Mockito.mock(EventQueue.class); + final PipelineActionHandler actionHandler = new PipelineActionHandler( + manager, SCMContext.emptyContext(), null); + final Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + + actionHandler.onMessage(getPipelineActionsFromDatanode( + pipeline.getId()), queue); + Mockito.verify(manager, Mockito.times(1)) + .closePipeline(pipeline.getId()); + } + @Test - public void testCloseActionForMissingPipeline() + public void testPipelineActionHandlerForValidPipelineInFollower() throws IOException { final PipelineManager manager = Mockito.mock(PipelineManager.class); final EventQueue queue = Mockito.mock(EventQueue.class); + final SCMContext context = SCMContext.emptyContext(); + final PipelineActionHandler actionHandler = new PipelineActionHandler( + manager, context, null); + final Pipeline pipeline = HddsTestUtils.getRandomPipeline(); + + context.updateLeaderAndTerm(false, 1); + actionHandler.onMessage(getPipelineActionsFromDatanode( + pipeline.getId()), queue); + Mockito.verify(manager, Mockito.times(0)) + .closePipeline(pipeline.getId()); + Mockito.verify(queue, Mockito.times(0)) + .fireEvent(Mockito.eq(SCMEvents.DATANODE_COMMAND), + Mockito.any(CommandForDatanode.class)); + } + + @Test + public void testPipelineActionHandlerForUnknownPipeline() throws IOException { + final PipelineManager manager = Mockito.mock(PipelineManager.class); + final EventQueue queue = Mockito.mock(EventQueue.class); + final PipelineActionHandler actionHandler = new PipelineActionHandler( + manager, SCMContext.emptyContext(), null); + final Pipeline pipeline = HddsTestUtils.getRandomPipeline(); Mockito.doThrow(new PipelineNotFoundException()) - .when(manager).closePipeline(Mockito.any(PipelineID.class)); + .when(manager).closePipeline(pipeline.getId()); + actionHandler.onMessage(getPipelineActionsFromDatanode( + pipeline.getId()), queue); + Mockito.verify(queue, Mockito.times(1)) + .fireEvent(Mockito.eq(SCMEvents.DATANODE_COMMAND), + Mockito.any(CommandForDatanode.class)); + } + + @Test + public void testPipelineActionHandlerForUnknownPipelineInFollower() + throws IOException { + + final PipelineManager manager = Mockito.mock(PipelineManager.class); + final EventQueue queue = Mockito.mock(EventQueue.class); + final SCMContext context = SCMContext.emptyContext(); + final PipelineActionHandler actionHandler = new PipelineActionHandler( + manager, context, null); + final Pipeline pipeline = HddsTestUtils.getRandomPipeline(); - final PipelineActionHandler actionHandler = - new PipelineActionHandler(manager, SCMContext.emptyContext(), null); + context.updateLeaderAndTerm(false, 1); + Mockito.doThrow(new PipelineNotFoundException()) + .when(manager).closePipeline(pipeline.getId()); + actionHandler.onMessage(getPipelineActionsFromDatanode( + pipeline.getId()), queue); + Mockito.verify(queue, Mockito.times(0)) + .fireEvent(Mockito.eq(SCMEvents.DATANODE_COMMAND), + Mockito.any(CommandForDatanode.class)); + + } + private PipelineActionsFromDatanode getPipelineActionsFromDatanode( + PipelineID pipelineID) { final PipelineActionsProto actionsProto = PipelineActionsProto.newBuilder() .addPipelineActions(PipelineAction.newBuilder() - .setClosePipeline(ClosePipelineInfo.newBuilder() - .setPipelineID(HddsProtos.PipelineID.newBuilder() - .setId(UUID.randomUUID().toString()).build()) - .setReason(ClosePipelineInfo.Reason.PIPELINE_FAILED)) + .setClosePipeline(ClosePipelineInfo.newBuilder() + .setPipelineID(pipelineID.getProtobuf()) + .setReason(ClosePipelineInfo.Reason.PIPELINE_FAILED)) .setAction(PipelineAction.Action.CLOSE).build()) .build(); - final PipelineActionsFromDatanode pipelineActions = - new PipelineActionsFromDatanode( - MockDatanodeDetails.randomDatanodeDetails(), actionsProto); - - actionHandler.onMessage(pipelineActions, queue); - - Mockito.verify(queue, Mockito.times(1)) - .fireEvent(Mockito.any(), Mockito.any(CommandForDatanode.class)); - + return new PipelineActionsFromDatanode( + MockDatanodeDetails.randomDatanodeDetails(), actionsProto); } } From 1789b5d9a172cad02947401d64e5fd32094eb9b9 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Fri, 1 Dec 2023 20:19:32 +0530 Subject: [PATCH 43/51] HDDS-6627. Intermittent failure in TestGetCommittedBlockLengthAndPutKey (#5698) --- .../TestGetCommittedBlockLengthAndPutKey.java | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java index e503a48b5cf..f236c945715 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java @@ -42,12 +42,17 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.ozone.test.tag.Flaky; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; import static java.nio.charset.StandardCharsets.UTF_8; @@ -56,7 +61,8 @@ */ @Timeout(300) public class TestGetCommittedBlockLengthAndPutKey { - + private static final Logger LOG = + LoggerFactory.getLogger(TestGetCommittedBlockLengthAndPutKey.class); private static MiniOzoneCluster cluster; private static OzoneConfiguration ozoneConfig; private static StorageContainerLocationProtocolClientSideTranslatorPB @@ -85,9 +91,9 @@ public static void shutdown() throws InterruptedException { } @Test - @Flaky("HDDS-6627") public void tesGetCommittedBlockLength() throws Exception { - ContainerProtos.GetCommittedBlockLengthResponseProto response; + final AtomicReference + response = new AtomicReference<>(); ContainerWithPipeline container = storageContainerLocationClient .allocateContainer(SCMTestUtils.getReplicationType(ozoneConfig), HddsProtos.ReplicationFactor.ONE, OzoneConsts.OZONE); @@ -99,7 +105,7 @@ public void tesGetCommittedBlockLength() throws Exception { BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); byte[] data = - RandomStringUtils.random(RandomUtils.nextInt(0, 1024)).getBytes(UTF_8); + RandomStringUtils.random(RandomUtils.nextInt(1, 1024)).getBytes(UTF_8); ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper .getWriteChunkRequest(container.getPipeline(), blockID, @@ -110,12 +116,20 @@ public void tesGetCommittedBlockLength() throws Exception { ContainerTestHelper .getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk()); client.sendCommand(putKeyRequest); - response = ContainerProtocolCalls - .getCommittedBlockLength(client, blockID, null); + GenericTestUtils.waitFor(() -> { + try { + response.set(ContainerProtocolCalls + .getCommittedBlockLength(client, blockID, null)); + } catch (IOException e) { + LOG.debug("Ignore the exception till wait: {}", e.getMessage()); + return false; + } + return true; + }, 500, 5000); // make sure the block ids in the request and response are same. - Assertions.assertTrue( - BlockID.getFromProtobuf(response.getBlockID()).equals(blockID)); - Assertions.assertTrue(response.getBlockLength() == data.length); + Assertions.assertEquals(blockID, + BlockID.getFromProtobuf(response.get().getBlockID())); + Assertions.assertEquals(data.length, response.get().getBlockLength()); xceiverClientManager.releaseClient(client, false); } @@ -156,7 +170,7 @@ public void tesPutKeyResposne() throws Exception { BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); byte[] data = - RandomStringUtils.random(RandomUtils.nextInt(0, 1024)).getBytes(UTF_8); + RandomStringUtils.random(RandomUtils.nextInt(1, 1024)).getBytes(UTF_8); ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper .getWriteChunkRequest(container.getPipeline(), blockID, From 802fe5e21514405ce3ee4c5a2a737d409fe85ed7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:22:39 +0100 Subject: [PATCH 44/51] HDDS-9818. Ensure valid Raft log write buffer size (#5713) --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 -- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 -- .../src/main/resources/ozone-default.xml | 8 --- .../server/ratis/XceiverServerRatis.java | 54 ++++++++++--------- .../apache/hadoop/hdds/scm/ha/RatisUtil.java | 14 ++--- .../src/test/resources/ozone-site.xml | 12 +++++ .../om/ratis/OzoneManagerRatisServer.java | 2 + 7 files changed, 52 insertions(+), 46 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 60d4ac6ae1e..4b586b796d0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -67,10 +67,6 @@ public final class ScmConfigKeys { "dfs.container.ratis.segment.size"; public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT = "64MB"; - public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY = - "dfs.container.ratis.segment.buffer.size"; - public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT = - "2MB"; public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY = "dfs.container.ratis.segment.preallocated.size"; public static final String diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 338fd778b09..0b62b887a3c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -352,10 +352,6 @@ public final class OzoneConfigKeys { = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY; public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT; - public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY - = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY; - public static final String DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT - = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT; public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY; public static final String diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 3eaba5a5c6a..65019796b83 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -320,14 +320,6 @@ by Apache Ratis on datanodes. (64 MB by default) - - dfs.container.ratis.segment.buffer.size - 2MB - OZONE, RATIS, PERFORMANCE - The size of the raft segment buffer used - by Apache Ratis on datanodes. (2 MB by default) - - dfs.container.ratis.segment.preallocated.size 4MB diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 0f8f7d4eccd..c9ac85414a7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -103,7 +103,6 @@ import org.apache.ratis.server.RaftServerRpc; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.storage.RaftStorage; -import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.TraditionalBinaryPrefix; @@ -111,6 +110,13 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.hdds.DatanodeVersion.SEPARATE_RATIS_PORTS_AVAILABLE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY; +import static org.apache.ratis.util.Preconditions.assertTrue; /** * Creates a ratis server endpoint that acts as the communication layer for @@ -428,40 +434,40 @@ private long setRaftSegmentPreallocatedSize(RaftProperties properties) { OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT, StorageUnit.BYTES); - int logAppenderQueueNumElements = conf.getInt( - OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS, - OzoneConfigKeys - .DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT); - final int logAppenderQueueByteLimit = (int) conf.getStorageSize( - OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, - OzoneConfigKeys - .DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, - StorageUnit.BYTES); - RaftServerConfigKeys.Log.Appender - .setBufferElementLimit(properties, logAppenderQueueNumElements); - RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties, - SizeInBytes.valueOf(logAppenderQueueByteLimit)); RaftServerConfigKeys.Log.setPreallocatedSize(properties, SizeInBytes.valueOf(raftSegmentPreallocatedSize)); return raftSegmentPreallocatedSize; } private void setRaftSegmentAndWriteBufferSize(RaftProperties properties) { + final int logAppenderQueueNumElements = conf.getInt( + DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS, + DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT); + final int logAppenderQueueByteLimit = (int) conf.getStorageSize( + DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, + DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, + StorageUnit.BYTES); + final long raftSegmentSize = (long) conf.getStorageSize( - OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY, - OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT, + DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY, + DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT, StorageUnit.BYTES); + final long raftSegmentBufferSize = logAppenderQueueByteLimit + 8; + + assertTrue(raftSegmentBufferSize <= raftSegmentSize, + () -> DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT + " = " + + logAppenderQueueByteLimit + + " must be <= (" + DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY + " - 8" + + " = " + (raftSegmentSize - 8) + ")"); + + RaftServerConfigKeys.Log.Appender.setBufferElementLimit(properties, + logAppenderQueueNumElements); + RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties, + SizeInBytes.valueOf(logAppenderQueueByteLimit)); RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf(raftSegmentSize)); - final long raftSegmentBufferSize = (long) conf.getStorageSize( - OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_KEY, - OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_BUFFER_SIZE_DEFAULT, - StorageUnit.BYTES); RaftServerConfigKeys.Log.setWriteBufferSize(properties, - SizeInBytes.valueOf(raftSegmentBufferSize)); - Preconditions.assertTrue(raftSegmentBufferSize <= raftSegmentSize, - () -> "raftSegmentBufferSize = " + raftSegmentBufferSize - + " > raftSegmentSize = " + raftSegmentSize); + SizeInBytes.valueOf(raftSegmentBufferSize)); } private RpcType setRpcType(RaftProperties properties) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 4ae82d89bb0..ccef5aab24e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -173,12 +173,14 @@ private static void setRaftLogProperties(final RaftProperties properties, ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_NUM, ScmConfigKeys. OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_NUM_DEFAULT)); - Log.Appender.setBufferByteLimit(properties, SizeInBytes.valueOf( - (long) ozoneConf.getStorageSize( - ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT, - ScmConfigKeys. - OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, - StorageUnit.BYTES))); + final int logAppenderQueueByteLimit = (int) ozoneConf.getStorageSize( + ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT, + ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, + StorageUnit.BYTES); + Log.Appender.setBufferByteLimit(properties, + SizeInBytes.valueOf(logAppenderQueueByteLimit)); + Log.setWriteBufferSize(properties, + SizeInBytes.valueOf(logAppenderQueueByteLimit + 8)); Log.setPreallocatedSize(properties, SizeInBytes.valueOf( (long) ozoneConf.getStorageSize( ScmConfigKeys.OZONE_SCM_HA_RAFT_SEGMENT_PRE_ALLOCATED_SIZE, diff --git a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml index a27b7210e90..4c415c0c7e0 100644 --- a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml @@ -51,4 +51,16 @@ 1s + + dfs.container.ratis.log.appender.queue.byte-limit + 8MB + + + ozone.om.ratis.log.appender.queue.byte-limit + 4MB + + + ozone.scm.ha.ratis.log.appender.queue.byte-limit + 4MB + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index b5f460a85f2..f26a5e300aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -661,6 +661,8 @@ public static RaftProperties newRaftProperties(ConfigurationSource conf, logAppenderQueueNumElements); RaftServerConfigKeys.Log.Appender.setBufferByteLimit(properties, SizeInBytes.valueOf(logAppenderQueueByteLimit)); + RaftServerConfigKeys.Log.setWriteBufferSize(properties, + SizeInBytes.valueOf(logAppenderQueueByteLimit + 8)); RaftServerConfigKeys.Log.setPreallocatedSize(properties, SizeInBytes.valueOf(raftSegmentPreallocatedSize)); RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties, From 74de0705c916bb877eae44310f43a0d7069cdef9 Mon Sep 17 00:00:00 2001 From: XiChen <32928346+xichen01@users.noreply.github.com> Date: Sun, 3 Dec 2023 21:14:40 +0800 Subject: [PATCH 45/51] HDDS-9784. Reduce default log.appender.wait-time.min to 0us in Datanode (#5711) --- .../common/statemachine/TestDatanodeConfiguration.java | 8 ++++---- .../hadoop/hdds/conf/DatanodeRatisServerConfig.java | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java index 1d83c5c7a5c..abfa2c76147 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java @@ -183,13 +183,13 @@ public void testConf() throws Exception { final DatanodeRatisServerConfig ratisConf = conf.getObject( DatanodeRatisServerConfig.class); - Assertions.assertEquals(1, ratisConf.getLogAppenderWaitTimeMin(), + Assertions.assertEquals(0, ratisConf.getLogAppenderWaitTimeMin(), "getLogAppenderWaitTimeMin"); - assertWaitTimeMin(TimeDuration.ONE_MILLISECOND, conf); - ratisConf.setLogAppenderWaitTimeMin(0); - conf.setFromObject(ratisConf); assertWaitTimeMin(TimeDuration.ZERO, conf); + ratisConf.setLogAppenderWaitTimeMin(1); + conf.setFromObject(ratisConf); + assertWaitTimeMin(TimeDuration.ONE_MILLISECOND, conf); } static void assertWaitTimeMin(TimeDuration expected, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java index 3eb0faf3021..643c94ebc27 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java @@ -193,10 +193,14 @@ public void setPreVote(boolean preVote) { /** @see RaftServerConfigKeys.Log.Appender#WAIT_TIME_MIN_KEY */ @Config(key = "log.appender.wait-time.min", - defaultValue = "1ms", + defaultValue = "0us", type = ConfigType.TIME, tags = {OZONE, DATANODE, RATIS, PERFORMANCE}, - description = "Minimum wait time between two appendEntries calls." + description = "The minimum wait time between two appendEntries calls. " + + "In some error conditions, the leader may keep retrying " + + "appendEntries. If it happens, increasing this value to, say, " + + "5us (microseconds) can help avoid the leader being too busy " + + "retrying." ) private long logAppenderWaitTimeMin; From 6ed190c6eb3e06013567b247f7a26adaacf14392 Mon Sep 17 00:00:00 2001 From: XiChen <32928346+xichen01@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:31:22 +0800 Subject: [PATCH 46/51] HDDS-9822. Format audit log message lazily (#5718) --- .../hadoop/ozone/audit/AuditLogger.java | 10 ++++-- .../hadoop/ozone/audit/AuditMessage.java | 32 +++++++++---------- .../ozone/s3/endpoint/BucketEndpoint.java | 2 +- .../ozone/s3/endpoint/EndpointBase.java | 3 +- .../ozone/s3/endpoint/ObjectEndpoint.java | 8 ++--- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java index f4f8ba78537..33ad7086987 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java @@ -29,7 +29,9 @@ import java.util.Collection; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -52,6 +54,7 @@ public class AuditLogger { public static final String AUDIT_LOG_DEBUG_CMD_LIST_PREFIX = "ozone.audit.log.debug.cmd.list."; private AuditLoggerType type; + private final Map opNameCache = new ConcurrentHashMap<>(); /** * Parametrized Constructor to initialize logger. @@ -130,8 +133,11 @@ public void refreshDebugCmdSet(OzoneConfiguration conf) { } private boolean shouldLogAtDebug(AuditMessage auditMessage) { - return debugCmdSetRef.get() - .contains(auditMessage.getOp().toLowerCase(Locale.ROOT)); + return debugCmdSetRef.get().contains(getLowerCaseOp(auditMessage.getOp())); + } + + private String getLowerCaseOp(String op) { + return opNameCache.computeIfAbsent(op, k -> k.toLowerCase(Locale.ROOT)); } /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java index bff05f024de..b8156d58d92 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java @@ -16,38 +16,36 @@ */ package org.apache.hadoop.ozone.audit; +import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import org.apache.logging.log4j.message.Message; +import org.apache.ratis.util.MemoizedSupplier; import java.util.Map; +import java.util.function.Supplier; /** * Defines audit message structure. */ public final class AuditMessage implements Message { - private final String message; - private final String user; - private final String ip; + private static final long serialVersionUID = 1L; + + private final transient Supplier messageSupplier; private final String op; - private final Map params; - private final String ret; private final Throwable throwable; private AuditMessage(String user, String ip, String op, Map params, String ret, Throwable throwable, - String performance) { - this.user = user; - this.ip = ip; + PerformanceStringBuilder performance) { this.op = op; - this.params = params; - this.ret = ret; - this.message = formMessage(user, ip, op, params, ret, performance); + this.messageSupplier = MemoizedSupplier.valueOf( + () -> formMessage(user, ip, op, params, ret, performance)); this.throwable = throwable; } @Override public String getFormattedMessage() { - return message; + return messageSupplier.get(); } @Override @@ -79,7 +77,7 @@ public static class Builder { private String op; private Map params; private String ret; - private String performance; + private PerformanceStringBuilder performance; public Builder setUser(String usr) { this.user = usr; @@ -111,7 +109,7 @@ public Builder withException(Throwable ex) { return this; } - public Builder setPerformance(String perf) { + public Builder setPerformance(PerformanceStringBuilder perf) { this.performance = perf; return this; } @@ -124,9 +122,9 @@ public AuditMessage build() { private String formMessage(String userStr, String ipStr, String opStr, Map paramsMap, String retStr, - String performanceMap) { - String perf = performanceMap != null && !performanceMap.isEmpty() - ? " | perf=" + performanceMap : ""; + PerformanceStringBuilder performanceMap) { + String perf = performanceMap != null + ? " | perf=" + performanceMap.build() : ""; return "user=" + userStr + " | ip=" + ipStr + " | " + "op=" + opStr + " " + paramsMap + " | ret=" + retStr + perf; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 910b0026e31..f7ec7130eaf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -275,7 +275,7 @@ public Response get( perf.appendCount(keyCount); perf.appendOpLatencyNanos(opLatencyNs); AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf.build())); + getAuditParameters(), perf)); response.setKeyCount(keyCount); return Response.ok(response).build(); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index abda4678dca..6f0f3c48472 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder; import org.apache.hadoop.ozone.audit.AuditLoggerType; import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.audit.Auditor; @@ -354,7 +355,7 @@ public AuditMessage buildAuditMessageForSuccess(AuditAction op, } public AuditMessage buildAuditMessageForSuccess(AuditAction op, - Map auditMap, String performance) { + Map auditMap, PerformanceStringBuilder performance) { AuditMessage.Builder builder = auditMessageBaseBuilder(op, auditMap) .withResult(AuditEventStatus.SUCCESS); builder.setPerformance(performance); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b607b1c5cff..1e247c8eb85 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -370,7 +370,7 @@ public Response put( long opLatencyNs = getMetrics().updateCreateKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf.build())); + getAuditParameters(), perf)); } } } @@ -405,7 +405,7 @@ public Response get( Response response = listParts(bucketName, keyPath, uploadId, partMarker, maxParts, perf); AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, - getAuditParameters(), perf.build())); + getAuditParameters(), perf)); return response; } @@ -444,7 +444,7 @@ public Response get( long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, - getAuditParameters(), perf.build())); + getAuditParameters(), perf)); }; responseBuilder = Response .ok(output) @@ -468,7 +468,7 @@ public Response get( long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); perf.appendOpLatencyNanos(opLatencyNs); AUDIT.logReadSuccess(buildAuditMessageForSuccess(S3GAction.GET_KEY, - getAuditParameters(), perf.build())); + getAuditParameters(), perf)); }; responseBuilder = Response .status(Status.PARTIAL_CONTENT) From eb7e8d009e701d1dc7a792deca1b5d6893d051c0 Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Mon, 4 Dec 2023 01:10:39 -0800 Subject: [PATCH 47/51] HDDS-9668. Zero-Copy in EC GRPC (write) (#5621) --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 + .../src/main/resources/ozone-default.xml | 8 + .../transport/server/GrpcXceiverService.java | 74 ++- .../transport/server/XceiverServerGrpc.java | 11 +- .../rpc/AbstractTestECKeyOutputStream.java | 459 ++++++++++++++++++ .../client/rpc/TestECKeyOutputStream.java | 432 +---------------- .../TestECKeyOutputStreamWithZeroCopy.java | 31 ++ 7 files changed, 584 insertions(+), 435 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStreamWithZeroCopy.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 0b62b887a3c..89c9be54675 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -188,6 +188,10 @@ public final class OzoneConfigKeys { "ozone.client.ec.grpc.write.timeout"; public static final String OZONE_CLIENT_EC_GRPC_WRITE_TIMEOUT_DEFAULT = "30s"; + public static final String OZONE_EC_GRPC_ZERO_COPY_ENABLED = + "ozone.ec.grpc.zerocopy.enabled"; + public static final boolean OZONE_EC_GRPC_ZERO_COPY_ENABLED_DEFAULT = true; + /** * Ozone administrator users delimited by comma. * If not set, only the user who launches an ozone service will be the diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 65019796b83..7d8f5381789 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4248,4 +4248,12 @@ to existing buckets till this operation is completed. + + ozone.ec.grpc.zerocopy.enabled + true + OZONE, DATANODE + + Specify if zero-copy should be enabled for EC GRPC protocol. + + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java index 023786fdb0d..9c3f29d0f0c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java @@ -17,19 +17,25 @@ */ package org.apache.hadoop.ozone.container.common.transport.server; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.datanode.proto - .XceiverClientProtocolServiceGrpc; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; +import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller; +import org.apache.ratis.thirdparty.com.google.protobuf.MessageLite; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc.getSendMethod; + /** * Grpc Service for handling Container Commands on datanode. */ @@ -39,9 +45,58 @@ public class GrpcXceiverService extends LOG = LoggerFactory.getLogger(GrpcXceiverService.class); private final ContainerDispatcher dispatcher; + private final boolean zeroCopyEnabled; + private final ZeroCopyMessageMarshaller + zeroCopyMessageMarshaller = new ZeroCopyMessageMarshaller<>( + ContainerCommandRequestProto.getDefaultInstance()); - public GrpcXceiverService(ContainerDispatcher dispatcher) { + public GrpcXceiverService(ContainerDispatcher dispatcher, + boolean zeroCopyEnabled) { this.dispatcher = dispatcher; + this.zeroCopyEnabled = zeroCopyEnabled; + } + + /** + * Bind service with zerocopy marshaller equipped for the `send` API if + * zerocopy is enabled. + * @return service definition. + */ + public ServerServiceDefinition bindServiceWithZeroCopy() { + ServerServiceDefinition orig = super.bindService(); + if (!zeroCopyEnabled) { + LOG.info("Zerocopy is not enabled."); + return orig; + } + + ServerServiceDefinition.Builder builder = + ServerServiceDefinition.builder(orig.getServiceDescriptor().getName()); + // Add `send` method with zerocopy marshaller. + addZeroCopyMethod(orig, builder, getSendMethod(), + zeroCopyMessageMarshaller); + // Add other methods as is. + orig.getMethods().stream().filter( + x -> !x.getMethodDescriptor().getFullMethodName().equals( + getSendMethod().getFullMethodName()) + ).forEach( + builder::addMethod + ); + + return builder.build(); + } + + private static void addZeroCopyMethod( + ServerServiceDefinition orig, + ServerServiceDefinition.Builder newServiceBuilder, + MethodDescriptor origMethod, + ZeroCopyMessageMarshaller zeroCopyMarshaller) { + MethodDescriptor newMethod = origMethod.toBuilder() + .setRequestMarshaller(zeroCopyMarshaller) + .build(); + @SuppressWarnings("unchecked") + ServerCallHandler serverCallHandler = + (ServerCallHandler) orig.getMethod( + newMethod.getFullMethodName()).getServerCallHandler(); + newServiceBuilder.addMethod(newMethod, serverCallHandler); } @Override @@ -61,6 +116,11 @@ public void onNext(ContainerCommandRequestProto request) { + " ContainerCommandRequestProto {}", request, e); isClosed.set(true); responseObserver.onError(e); + } finally { + InputStream popStream = zeroCopyMessageMarshaller.popStream(request); + if (popStream != null) { + IOUtils.close(LOG, popStream); + } } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java index b421177b44c..009e6396e0d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java @@ -66,6 +66,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED_DEFAULT; + /** * Creates a Grpc server endpoint that acts as the communication layer for * Ozone containers. @@ -131,8 +134,13 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails, eventLoopGroup = new NioEventLoopGroup(poolSize / 10, factory); channelType = NioServerSocketChannel.class; } + final boolean zeroCopyEnabled = conf.getBoolean( + OZONE_EC_GRPC_ZERO_COPY_ENABLED, + OZONE_EC_GRPC_ZERO_COPY_ENABLED_DEFAULT); LOG.info("GrpcServer channel type {}", channelType.getSimpleName()); + GrpcXceiverService xceiverService = new GrpcXceiverService(dispatcher, + zeroCopyEnabled); NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port) .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) .bossEventLoopGroup(eventLoopGroup) @@ -140,7 +148,8 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails, .channelType(channelType) .executor(readExecutors) .addService(ServerInterceptors.intercept( - new GrpcXceiverService(dispatcher), new GrpcServerInterceptor())); + xceiverService.bindServiceWithZeroCopy(), + new GrpcServerInterceptor())); SecurityConfig secConf = new SecurityConfig(conf); if (secConf.isSecurityEnabled() && secConf.isGrpcTlsEnabled()) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java new file mode 100644 index 00000000000..518893aa0a0 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java @@ -0,0 +1,459 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.client.rpc; + +import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; +import org.apache.hadoop.ozone.client.io.KeyOutputStream; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.container.TestHelper; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; + +/** + * Tests key output stream. + */ +abstract class AbstractTestECKeyOutputStream { + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf = new OzoneConfiguration(); + private static OzoneClient client; + private static ObjectStore objectStore; + private static int chunkSize; + private static int flushSize; + private static int maxFlushSize; + private static int blockSize; + private static String volumeName; + private static String bucketName; + private static String keyString; + private static int dataBlocks = 3; + private static int inputSize = dataBlocks * chunkSize; + private static byte[][] inputChunks = new byte[dataBlocks][chunkSize]; + + /** + * Create a MiniDFSCluster for testing. + */ + protected static void init(boolean zeroCopyEnabled) throws Exception { + chunkSize = 1024 * 1024; + flushSize = 2 * chunkSize; + maxFlushSize = 2 * flushSize; + blockSize = 2 * maxFlushSize; + + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setChecksumType(ContainerProtos.ChecksumType.NONE); + clientConfig.setStreamBufferFlushDelay(false); + conf.setFromObject(clientConfig); + + // If SCM detects dead node too quickly, then container would be moved to + // closed state and all in progress writes will get exception. To avoid + // that, we are just keeping higher timeout and none of the tests depending + // on deadnode detection timeout currently. + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); + conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 60, TimeUnit.SECONDS); + conf.setTimeDuration("hdds.ratis.raft.server.rpc.slowness.timeout", 300, + TimeUnit.SECONDS); + conf.setTimeDuration( + "hdds.ratis.raft.server.notification.no-leader.timeout", 300, + TimeUnit.SECONDS); + conf.setQuietMode(false); + conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, + StorageUnit.MB); + conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 500, + TimeUnit.MILLISECONDS); + conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, + TimeUnit.SECONDS); + conf.setBoolean(OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED, + zeroCopyEnabled); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) + .setTotalPipelineNumLimit(10).setBlockSize(blockSize) + .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) + .setStreamBufferMaxSize(maxFlushSize) + .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); + cluster.waitForClusterToBeReady(); + client = OzoneClientFactory.getRpcClient(conf); + objectStore = client.getObjectStore(); + keyString = UUID.randomUUID().toString(); + volumeName = "testeckeyoutputstream"; + bucketName = volumeName; + objectStore.createVolume(volumeName); + objectStore.getVolume(volumeName).createBucket(bucketName); + initInputChunks(); + } + + @BeforeClass + public static void init() throws Exception { + init(false); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + IOUtils.closeQuietly(client); + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testCreateKeyWithECReplicationConfig() throws Exception { + try (OzoneOutputStream key = TestHelper + .createKey(keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, chunkSize), inputSize, + objectStore, volumeName, bucketName)) { + Assert.assertTrue(key.getOutputStream() instanceof ECKeyOutputStream); + } + } + + @Test + public void testCreateKeyWithOutBucketDefaults() throws Exception { + OzoneVolume volume = objectStore.getVolume(volumeName); + OzoneBucket bucket = volume.getBucket(bucketName); + try (OzoneOutputStream out = bucket.createKey("myKey", inputSize)) { + Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + } + + @Test + public void testCreateKeyWithBucketDefaults() throws Exception { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + bucketArgs.setDefaultReplicationConfig( + new DefaultReplicationConfig( + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize))); + + volume.createBucket(myBucket, bucketArgs.build()); + OzoneBucket bucket = volume.getBucket(myBucket); + + try (OzoneOutputStream out = bucket.createKey(keyString, inputSize)) { + Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + byte[] buf = new byte[chunkSize]; + try (OzoneInputStream in = bucket.readKey(keyString)) { + for (int i = 0; i < inputChunks.length; i++) { + int read = in.read(buf, 0, chunkSize); + Assert.assertEquals(chunkSize, read); + Assert.assertTrue(Arrays.equals(buf, inputChunks[i])); + } + } + } + + @Test + public void testOverwriteECKeyWithRatisKey() throws Exception { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + volume.createBucket(myBucket, bucketArgs.build()); + OzoneBucket bucket = volume.getBucket(myBucket); + createKeyAndCheckReplicationConfig(keyString, bucket, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize)); + + //Overwrite with RATIS/THREE + createKeyAndCheckReplicationConfig(keyString, bucket, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); + + //Overwrite with RATIS/ONE + createKeyAndCheckReplicationConfig(keyString, bucket, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); + } + + @Test + public void testOverwriteRatisKeyWithECKey() throws Exception { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + volume.createBucket(myBucket, bucketArgs.build()); + OzoneBucket bucket = volume.getBucket(myBucket); + + createKeyAndCheckReplicationConfig(keyString, bucket, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); + // Overwrite with EC key + createKeyAndCheckReplicationConfig(keyString, bucket, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize)); + } + + private void createKeyAndCheckReplicationConfig(String keyName, + OzoneBucket bucket, ReplicationConfig replicationConfig) + throws IOException { + try (OzoneOutputStream out = bucket + .createKey(keyName, inputSize, replicationConfig, new HashMap<>())) { + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + OzoneKeyDetails key = bucket.getKey(keyName); + Assert.assertEquals(replicationConfig, key.getReplicationConfig()); + } + + @Test + public void testCreateRatisKeyAndWithECBucketDefaults() throws Exception { + OzoneBucket bucket = getOzoneBucket(); + try (OzoneOutputStream out = bucket.createKey( + "testCreateRatisKeyAndWithECBucketDefaults", 2000, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + new HashMap<>())) { + Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + } + + @Test + public void test13ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(13); + } + + @Test + public void testChunksInSingleWriteOpWithOffset() throws IOException { + testMultipleChunksInSingleWriteOp(11, 25, 19); + } + + @Test + public void test15ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(15); + } + + @Test + public void test20ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(20); + } + + @Test + public void test21ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(21); + } + + private void testMultipleChunksInSingleWriteOp(int offset, + int bufferChunks, int numChunks) + throws IOException { + byte[] inputData = getInputBytes(offset, bufferChunks, numChunks); + final OzoneBucket bucket = getOzoneBucket(); + String keyName = + String.format("testMultipleChunksInSingleWriteOpOffset" + + "%dBufferChunks%dNumChunks", offset, bufferChunks, + numChunks); + try (OzoneOutputStream out = bucket.createKey(keyName, 4096, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize), new HashMap<>())) { + out.write(inputData, offset, numChunks * chunkSize); + } + + validateContent(offset, numChunks * chunkSize, inputData, bucket, + bucket.getKey(keyName)); + } + + private void testMultipleChunksInSingleWriteOp(int numChunks) + throws IOException { + testMultipleChunksInSingleWriteOp(0, numChunks, numChunks); + } + + @Test + public void testECContainerKeysCountAndNumContainerReplicas() + throws IOException, InterruptedException, TimeoutException { + byte[] inputData = getInputBytes(1); + final OzoneBucket bucket = getOzoneBucket(); + ContainerOperationClient containerOperationClient = + new ContainerOperationClient(conf); + + ECReplicationConfig repConfig = new ECReplicationConfig( + 3, 2, ECReplicationConfig.EcCodec.RS, chunkSize); + // Close all EC pipelines so we must get a fresh pipeline and hence + // container for this test. + PipelineManager pm = + cluster.getStorageContainerManager().getPipelineManager(); + for (Pipeline p : pm.getPipelines(repConfig)) { + pm.closePipeline(p, true); + } + + String keyName = UUID.randomUUID().toString(); + try (OzoneOutputStream out = bucket.createKey(keyName, 4096, + repConfig, new HashMap<>())) { + out.write(inputData); + } + OzoneKeyDetails key = bucket.getKey(keyName); + long currentKeyContainerID = + key.getOzoneKeyLocations().get(0).getContainerID(); + + GenericTestUtils.waitFor(() -> { + try { + return (containerOperationClient.getContainer(currentKeyContainerID) + .getNumberOfKeys() == 1) && (containerOperationClient + .getContainerReplicas(currentKeyContainerID).size() == 5); + } catch (IOException exception) { + Assert.fail("Unexpected exception " + exception); + return false; + } + }, 100, 10000); + validateContent(inputData, bucket, key); + } + + private void validateContent(byte[] inputData, OzoneBucket bucket, + OzoneKey key) throws IOException { + validateContent(0, inputData.length, inputData, bucket, key); + } + + private void validateContent(int offset, int length, byte[] inputData, + OzoneBucket bucket, + OzoneKey key) throws IOException { + try (OzoneInputStream is = bucket.readKey(key.getName())) { + byte[] fileContent = new byte[length]; + Assert.assertEquals(length, is.read(fileContent)); + Assert.assertEquals(new String(Arrays.copyOfRange(inputData, offset, + offset + length), UTF_8), + new String(fileContent, UTF_8)); + } + } + + private OzoneBucket getOzoneBucket() throws IOException { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + bucketArgs.setDefaultReplicationConfig( + new DefaultReplicationConfig( + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize))); + + volume.createBucket(myBucket, bucketArgs.build()); + return volume.getBucket(myBucket); + } + + private static void initInputChunks() { + for (int i = 0; i < dataBlocks; i++) { + inputChunks[i] = getBytesWith(i + 1, chunkSize); + } + } + + private static byte[] getBytesWith(int singleDigitNumber, int total) { + StringBuilder builder = new StringBuilder(singleDigitNumber); + for (int i = 1; i <= total; i++) { + builder.append(singleDigitNumber); + } + return builder.toString().getBytes(UTF_8); + } + + @Test + public void testWriteShouldSucceedWhenDNKilled() throws Exception { + int numChunks = 3; + byte[] inputData = getInputBytes(numChunks); + final OzoneBucket bucket = getOzoneBucket(); + String keyName = "testWriteShouldSucceedWhenDNKilled" + numChunks; + DatanodeDetails nodeToKill = null; + try { + try (OzoneOutputStream out = bucket.createKey(keyName, 1024, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize), new HashMap<>())) { + ECKeyOutputStream ecOut = (ECKeyOutputStream) out.getOutputStream(); + out.write(inputData); + // Kill a node from first pipeline + nodeToKill = ecOut.getStreamEntries() + .get(0).getPipeline().getFirstNode(); + cluster.shutdownHddsDatanode(nodeToKill); + + out.write(inputData); + + // Wait for flushing thread to finish its work. + final long checkpoint = System.currentTimeMillis(); + ecOut.insertFlushCheckpoint(checkpoint); + GenericTestUtils.waitFor(() -> ecOut.getFlushCheckpoint() == checkpoint, + 100, 10000); + + // Check the second blockGroup pipeline to make sure that the failed + // node is not selected. + Assert.assertFalse(ecOut.getStreamEntries() + .get(1).getPipeline().getNodes().contains(nodeToKill)); + } + + try (OzoneInputStream is = bucket.readKey(keyName)) { + // We wrote "inputData" twice, so do two reads and ensure the correct + // data comes back. + for (int i = 0; i < 2; i++) { + byte[] fileContent = new byte[inputData.length]; + Assert.assertEquals(inputData.length, is.read(fileContent)); + Assert.assertEquals(new String(inputData, UTF_8), + new String(fileContent, UTF_8)); + } + } + } finally { + cluster.restartHddsDatanode(nodeToKill, true); + } + } + + private byte[] getInputBytes(int numChunks) { + return getInputBytes(0, numChunks, numChunks); + } + + private byte[] getInputBytes(int offset, int bufferChunks, int numChunks) { + byte[] inputData = new byte[offset + bufferChunks * chunkSize]; + for (int i = 0; i < numChunks; i++) { + int start = offset + (i * chunkSize); + Arrays.fill(inputData, start, start + chunkSize - 1, + String.valueOf(i % 9).getBytes(UTF_8)[0]); + } + return inputData; + } + +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java index a429b94d5a7..dc5622e1e8b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java @@ -14,440 +14,18 @@ * License for the specific language governing permissions and limitations under * the License. */ + package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.DefaultReplicationConfig; -import org.apache.hadoop.hdds.client.ECReplicationConfig; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; -import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.client.BucketArgs; -import org.apache.hadoop.ozone.client.ObjectStore; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; -import org.apache.hadoop.ozone.client.io.KeyOutputStream; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.TestHelper; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; /** - * Tests key output stream. + * Tests key output stream without zero-copy enabled. */ -public class TestECKeyOutputStream { - private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf = new OzoneConfiguration(); - private static OzoneClient client; - private static ObjectStore objectStore; - private static int chunkSize; - private static int flushSize; - private static int maxFlushSize; - private static int blockSize; - private static String volumeName; - private static String bucketName; - private static String keyString; - private static int dataBlocks = 3; - private static int inputSize = dataBlocks * chunkSize; - private static byte[][] inputChunks = new byte[dataBlocks][chunkSize]; - - /** - * Create a MiniDFSCluster for testing. - */ +public class TestECKeyOutputStream extends + AbstractTestECKeyOutputStream { @BeforeClass public static void init() throws Exception { - chunkSize = 1024 * 1024; - flushSize = 2 * chunkSize; - maxFlushSize = 2 * flushSize; - blockSize = 2 * maxFlushSize; - - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setChecksumType(ContainerProtos.ChecksumType.NONE); - clientConfig.setStreamBufferFlushDelay(false); - conf.setFromObject(clientConfig); - - // If SCM detects dead node too quickly, then container would be moved to - // closed state and all in progress writes will get exception. To avoid - // that, we are just keeping higher timeout and none of the tests depending - // on deadnode detection timeout currently. - conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 60, TimeUnit.SECONDS); - conf.setTimeDuration("hdds.ratis.raft.server.rpc.slowness.timeout", 300, - TimeUnit.SECONDS); - conf.setTimeDuration( - "hdds.ratis.raft.server.notification.no-leader.timeout", 300, - TimeUnit.SECONDS); - conf.setQuietMode(false); - conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, - StorageUnit.MB); - conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 500, - TimeUnit.MILLISECONDS); - conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, - TimeUnit.SECONDS); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) - .setTotalPipelineNumLimit(10).setBlockSize(blockSize) - .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) - .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); - cluster.waitForClusterToBeReady(); - client = OzoneClientFactory.getRpcClient(conf); - objectStore = client.getObjectStore(); - keyString = UUID.randomUUID().toString(); - volumeName = "testeckeyoutputstream"; - bucketName = volumeName; - objectStore.createVolume(volumeName); - objectStore.getVolume(volumeName).createBucket(bucketName); - initInputChunks(); - } - - /** - * Shutdown MiniDFSCluster. - */ - @AfterClass - public static void shutdown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } - - @Test - public void testCreateKeyWithECReplicationConfig() throws Exception { - try (OzoneOutputStream key = TestHelper - .createKey(keyString, new ECReplicationConfig(3, 2, - ECReplicationConfig.EcCodec.RS, chunkSize), inputSize, - objectStore, volumeName, bucketName)) { - Assert.assertTrue(key.getOutputStream() instanceof ECKeyOutputStream); - } - } - - @Test - public void testCreateKeyWithOutBucketDefaults() throws Exception { - OzoneVolume volume = objectStore.getVolume(volumeName); - OzoneBucket bucket = volume.getBucket(bucketName); - try (OzoneOutputStream out = bucket.createKey("myKey", inputSize)) { - Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); - for (int i = 0; i < inputChunks.length; i++) { - out.write(inputChunks[i]); - } - } - } - - @Test - public void testCreateKeyWithBucketDefaults() throws Exception { - String myBucket = UUID.randomUUID().toString(); - OzoneVolume volume = objectStore.getVolume(volumeName); - final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); - bucketArgs.setDefaultReplicationConfig( - new DefaultReplicationConfig( - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize))); - - volume.createBucket(myBucket, bucketArgs.build()); - OzoneBucket bucket = volume.getBucket(myBucket); - - try (OzoneOutputStream out = bucket.createKey(keyString, inputSize)) { - Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream); - for (int i = 0; i < inputChunks.length; i++) { - out.write(inputChunks[i]); - } - } - byte[] buf = new byte[chunkSize]; - try (OzoneInputStream in = bucket.readKey(keyString)) { - for (int i = 0; i < inputChunks.length; i++) { - int read = in.read(buf, 0, chunkSize); - Assert.assertEquals(chunkSize, read); - Assert.assertTrue(Arrays.equals(buf, inputChunks[i])); - } - } - } - - @Test - public void testOverwriteECKeyWithRatisKey() throws Exception { - String myBucket = UUID.randomUUID().toString(); - OzoneVolume volume = objectStore.getVolume(volumeName); - final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); - volume.createBucket(myBucket, bucketArgs.build()); - OzoneBucket bucket = volume.getBucket(myBucket); - createKeyAndCheckReplicationConfig(keyString, bucket, - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize)); - - //Overwrite with RATIS/THREE - createKeyAndCheckReplicationConfig(keyString, bucket, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); - - //Overwrite with RATIS/ONE - createKeyAndCheckReplicationConfig(keyString, bucket, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); - } - - @Test - public void testOverwriteRatisKeyWithECKey() throws Exception { - String myBucket = UUID.randomUUID().toString(); - OzoneVolume volume = objectStore.getVolume(volumeName); - final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); - volume.createBucket(myBucket, bucketArgs.build()); - OzoneBucket bucket = volume.getBucket(myBucket); - - createKeyAndCheckReplicationConfig(keyString, bucket, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); - // Overwrite with EC key - createKeyAndCheckReplicationConfig(keyString, bucket, - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize)); - } - - private void createKeyAndCheckReplicationConfig(String keyName, - OzoneBucket bucket, ReplicationConfig replicationConfig) - throws IOException { - try (OzoneOutputStream out = bucket - .createKey(keyName, inputSize, replicationConfig, new HashMap<>())) { - for (int i = 0; i < inputChunks.length; i++) { - out.write(inputChunks[i]); - } - } - OzoneKeyDetails key = bucket.getKey(keyName); - Assert.assertEquals(replicationConfig, key.getReplicationConfig()); - } - - @Test - public void testCreateRatisKeyAndWithECBucketDefaults() throws Exception { - OzoneBucket bucket = getOzoneBucket(); - try (OzoneOutputStream out = bucket.createKey( - "testCreateRatisKeyAndWithECBucketDefaults", 2000, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - new HashMap<>())) { - Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); - for (int i = 0; i < inputChunks.length; i++) { - out.write(inputChunks[i]); - } - } - } - - @Test - public void test13ChunksInSingleWriteOp() throws IOException { - testMultipleChunksInSingleWriteOp(13); - } - - @Test - public void testChunksInSingleWriteOpWithOffset() throws IOException { - testMultipleChunksInSingleWriteOp(11, 25, 19); - } - - @Test - public void test15ChunksInSingleWriteOp() throws IOException { - testMultipleChunksInSingleWriteOp(15); - } - - @Test - public void test20ChunksInSingleWriteOp() throws IOException { - testMultipleChunksInSingleWriteOp(20); + init(false); } - - @Test - public void test21ChunksInSingleWriteOp() throws IOException { - testMultipleChunksInSingleWriteOp(21); - } - - private void testMultipleChunksInSingleWriteOp(int offset, - int bufferChunks, int numChunks) - throws IOException { - byte[] inputData = getInputBytes(offset, bufferChunks, numChunks); - final OzoneBucket bucket = getOzoneBucket(); - String keyName = - String.format("testMultipleChunksInSingleWriteOpOffset" + - "%dBufferChunks%dNumChunks", offset, bufferChunks, - numChunks); - try (OzoneOutputStream out = bucket.createKey(keyName, 4096, - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize), new HashMap<>())) { - out.write(inputData, offset, numChunks * chunkSize); - } - - validateContent(offset, numChunks * chunkSize, inputData, bucket, - bucket.getKey(keyName)); - } - - private void testMultipleChunksInSingleWriteOp(int numChunks) - throws IOException { - testMultipleChunksInSingleWriteOp(0, numChunks, numChunks); - } - - @Test - public void testECContainerKeysCountAndNumContainerReplicas() - throws IOException, InterruptedException, TimeoutException { - byte[] inputData = getInputBytes(1); - final OzoneBucket bucket = getOzoneBucket(); - ContainerOperationClient containerOperationClient = - new ContainerOperationClient(conf); - - ECReplicationConfig repConfig = new ECReplicationConfig( - 3, 2, ECReplicationConfig.EcCodec.RS, chunkSize); - // Close all EC pipelines so we must get a fresh pipeline and hence - // container for this test. - PipelineManager pm = - cluster.getStorageContainerManager().getPipelineManager(); - for (Pipeline p : pm.getPipelines(repConfig)) { - pm.closePipeline(p, true); - } - - String keyName = UUID.randomUUID().toString(); - try (OzoneOutputStream out = bucket.createKey(keyName, 4096, - repConfig, new HashMap<>())) { - out.write(inputData); - } - OzoneKeyDetails key = bucket.getKey(keyName); - long currentKeyContainerID = - key.getOzoneKeyLocations().get(0).getContainerID(); - - GenericTestUtils.waitFor(() -> { - try { - return (containerOperationClient.getContainer(currentKeyContainerID) - .getNumberOfKeys() == 1) && (containerOperationClient - .getContainerReplicas(currentKeyContainerID).size() == 5); - } catch (IOException exception) { - Assert.fail("Unexpected exception " + exception); - return false; - } - }, 100, 10000); - validateContent(inputData, bucket, key); - } - - private void validateContent(byte[] inputData, OzoneBucket bucket, - OzoneKey key) throws IOException { - validateContent(0, inputData.length, inputData, bucket, key); - } - - private void validateContent(int offset, int length, byte[] inputData, - OzoneBucket bucket, - OzoneKey key) throws IOException { - try (OzoneInputStream is = bucket.readKey(key.getName())) { - byte[] fileContent = new byte[length]; - Assert.assertEquals(length, is.read(fileContent)); - Assert.assertEquals(new String(Arrays.copyOfRange(inputData, offset, - offset + length), UTF_8), - new String(fileContent, UTF_8)); - } - } - - private OzoneBucket getOzoneBucket() throws IOException { - String myBucket = UUID.randomUUID().toString(); - OzoneVolume volume = objectStore.getVolume(volumeName); - final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); - bucketArgs.setDefaultReplicationConfig( - new DefaultReplicationConfig( - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize))); - - volume.createBucket(myBucket, bucketArgs.build()); - return volume.getBucket(myBucket); - } - - private static void initInputChunks() { - for (int i = 0; i < dataBlocks; i++) { - inputChunks[i] = getBytesWith(i + 1, chunkSize); - } - } - - private static byte[] getBytesWith(int singleDigitNumber, int total) { - StringBuilder builder = new StringBuilder(singleDigitNumber); - for (int i = 1; i <= total; i++) { - builder.append(singleDigitNumber); - } - return builder.toString().getBytes(UTF_8); - } - - @Test - public void testWriteShouldSucceedWhenDNKilled() throws Exception { - int numChunks = 3; - byte[] inputData = getInputBytes(numChunks); - final OzoneBucket bucket = getOzoneBucket(); - String keyName = "testWriteShouldSucceedWhenDNKilled" + numChunks; - DatanodeDetails nodeToKill = null; - try { - try (OzoneOutputStream out = bucket.createKey(keyName, 1024, - new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, - chunkSize), new HashMap<>())) { - ECKeyOutputStream ecOut = (ECKeyOutputStream) out.getOutputStream(); - out.write(inputData); - // Kill a node from first pipeline - nodeToKill = ecOut.getStreamEntries() - .get(0).getPipeline().getFirstNode(); - cluster.shutdownHddsDatanode(nodeToKill); - - out.write(inputData); - - // Wait for flushing thread to finish its work. - final long checkpoint = System.currentTimeMillis(); - ecOut.insertFlushCheckpoint(checkpoint); - GenericTestUtils.waitFor(() -> ecOut.getFlushCheckpoint() == checkpoint, - 100, 10000); - - // Check the second blockGroup pipeline to make sure that the failed - // node is not selected. - Assert.assertFalse(ecOut.getStreamEntries() - .get(1).getPipeline().getNodes().contains(nodeToKill)); - } - - try (OzoneInputStream is = bucket.readKey(keyName)) { - // We wrote "inputData" twice, so do two reads and ensure the correct - // data comes back. - for (int i = 0; i < 2; i++) { - byte[] fileContent = new byte[inputData.length]; - Assert.assertEquals(inputData.length, is.read(fileContent)); - Assert.assertEquals(new String(inputData, UTF_8), - new String(fileContent, UTF_8)); - } - } - } finally { - cluster.restartHddsDatanode(nodeToKill, true); - } - } - - private byte[] getInputBytes(int numChunks) { - return getInputBytes(0, numChunks, numChunks); - } - - private byte[] getInputBytes(int offset, int bufferChunks, int numChunks) { - byte[] inputData = new byte[offset + bufferChunks * chunkSize]; - for (int i = 0; i < numChunks; i++) { - int start = offset + (i * chunkSize); - Arrays.fill(inputData, start, start + chunkSize - 1, - String.valueOf(i % 9).getBytes(UTF_8)[0]); - } - return inputData; - } - } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStreamWithZeroCopy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStreamWithZeroCopy.java new file mode 100644 index 00000000000..b9baeb2437f --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStreamWithZeroCopy.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.client.rpc; + +import org.junit.BeforeClass; + +/** + * Tests key output stream with zero-copy enabled. + */ +public class TestECKeyOutputStreamWithZeroCopy extends + AbstractTestECKeyOutputStream { + @BeforeClass + public static void init() throws Exception { + init(true); + } +} From b1588e82b586a143d59c5e94fee28f818db287a8 Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Mon, 4 Dec 2023 09:25:26 -0800 Subject: [PATCH 48/51] HDDS-9709. Avoid empty pipelines in the container pipeline cache. (#5625) --- .../om/TestOmContainerLocationCache.java | 89 ++++++++++++++++++- .../org/apache/hadoop/ozone/om/ScmClient.java | 10 ++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java index 2c9506d530c..c4948d404b5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java @@ -102,7 +102,9 @@ import static com.google.common.collect.Sets.newHashSet; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_REPLICA_FOUND; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -196,6 +198,11 @@ private static XceiverClientManager mockDataNodeClientFactory() mockDn1Protocol = spy(new XceiverClientGrpc(createPipeline(DN1), conf)); mockDn2Protocol = spy(new XceiverClientGrpc(createPipeline(DN2), conf)); XceiverClientManager manager = mock(XceiverClientManager.class); + when(manager.acquireClient(argThat(matchEmptyPipeline()))) + .thenCallRealMethod(); + when(manager.acquireClientForReadData(argThat(matchEmptyPipeline()))) + .thenCallRealMethod(); + when(manager.acquireClient(argThat(matchPipeline(DN1)))) .thenReturn(mockDn1Protocol); when(manager.acquireClientForReadData(argThat(matchPipeline(DN1)))) @@ -208,8 +215,14 @@ private static XceiverClientManager mockDataNodeClientFactory() return manager; } - private static ArgumentMatcher matchPipeline(DatanodeDetails dn) { + private static ArgumentMatcher matchEmptyPipeline() { return argument -> argument != null + && argument.getNodes().isEmpty(); + } + + + private static ArgumentMatcher matchPipeline(DatanodeDetails dn) { + return argument -> argument != null && !argument.getNodes().isEmpty() && argument.getNodes().get(0).getUuid().equals(dn.getUuid()); } @@ -500,6 +513,60 @@ public void containerNotRefreshedAfterDatanodeReadChunkError( .getContainerWithPipelineBatch(newHashSet(CONTAINER_ID.get())); } + /** + * Verify that in situation that SCM returns empty pipelines (that prevents + * clients from reading data), the empty pipelines are not cached and + * subsequent key reads re-fetch container data from SCM. + */ + @Test + public void containerRefreshedOnEmptyPipelines() throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(CONTAINER_ID.get(), 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, CONTAINER_ID.get(), 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + String keyName = "key"; + try (OzoneOutputStream os = bucket.createKey(keyName, data.length)) { + IOUtils.write(data, os); + } + + // All datanodes go down and scm returns empty pipeline for the container. + mockScmGetContainerPipelineEmpty(CONTAINER_ID.get()); + + OzoneKeyDetails key1 = bucket.getKey(keyName); + + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(CONTAINER_ID.get())); + + // verify that the effort to read will result in a NO_REPLICA_FOUND error. + Exception ex = + assertThrows(IllegalArgumentException.class, () -> { + try (InputStream is = key1.getContent()) { + IOUtils.read(is, new byte[(int) key1.getDataSize()]); + } + }); + assertEquals(NO_REPLICA_FOUND.toString(), ex.getMessage()); + + // but the empty pipeline is not cached, and when some data node is back. + mockScmGetContainerPipeline(CONTAINER_ID.get(), DN1); + mockGetBlock(mockDn1Protocol, CONTAINER_ID.get(), 1L, data, null, null); + mockReadChunk(mockDn1Protocol, CONTAINER_ID.get(), 1L, data, null, null); + // the subsequent effort to read the key is success. + OzoneKeyDetails updatedKey1 = bucket.getKey(keyName); + try (InputStream is = updatedKey1.getContent()) { + byte[] read = new byte[(int) key1.getDataSize()]; + IOUtils.read(is, read); + Assertions.assertArrayEquals(data, read); + } + // verify SCM is called one more time to refetch the container pipeline.. + verify(mockScmContainerClient, times(2)) + .getContainerWithPipelineBatch(newHashSet(CONTAINER_ID.get())); + } + private void mockPutBlockResponse(XceiverClientSpi mockDnProtocol, long containerId, long localId, byte[] data) @@ -578,6 +645,20 @@ private void mockScmGetContainerPipeline(long containerId, newHashSet(containerId))).thenReturn(containerWithPipelines); } + private void mockScmGetContainerPipelineEmpty(long containerId) + throws IOException { + Pipeline pipeline = createPipeline(Collections.emptyList()); + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(containerId) + .setPipelineID(pipeline.getId()).build(); + List containerWithPipelines = + Collections.singletonList( + new ContainerWithPipeline(containerInfo, pipeline)); + + when(mockScmContainerClient.getContainerWithPipelineBatch( + newHashSet(containerId))).thenReturn(containerWithPipelines); + } + private void mockGetBlock(XceiverClientGrpc mockDnProtocol, long containerId, long localId, byte[] data, @@ -664,12 +745,16 @@ private void mockReadChunk(XceiverClientGrpc mockDnProtocol, } private static Pipeline createPipeline(DatanodeDetails dn) { + return createPipeline(Collections.singletonList(dn)); + } + + private static Pipeline createPipeline(List nodes) { return Pipeline.newBuilder() .setState(Pipeline.PipelineState.OPEN) .setId(PipelineID.randomId()) .setReplicationConfig( RatisReplicationConfig.getInstance(ReplicationFactor.THREE)) - .setNodes(Collections.singletonList(dn)) + .setNodes(nodes) .build(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 0718c89c422..3a15f2e8d54 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -111,7 +112,14 @@ public Map getContainerLocations(Iterable containerIds, containerLocationCache.invalidateAll(containerIds); } try { - return containerLocationCache.getAll(containerIds); + Map result = containerLocationCache.getAll(containerIds); + // Don't keep empty pipelines in the cache. + List emptyPipelines = result.entrySet().stream() + .filter(e -> e.getValue().isEmpty()) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + containerLocationCache.invalidateAll(emptyPipelines); + return result; } catch (ExecutionException e) { return handleCacheExecutionException(e); } catch (InvalidCacheLoadException e) { From a4bda5c6a2da921b53d02c622f1b9d14c190fb91 Mon Sep 17 00:00:00 2001 From: Raju Balpande <146973984+raju-balpande@users.noreply.github.com> Date: Tue, 5 Dec 2023 00:30:13 +0530 Subject: [PATCH 49/51] HDDS-9775. Migrate simple OM integration tests to JUnit5 (#5723) --- .../om/TestBucketLayoutWithOlderClient.java | 34 ++--- .../hadoop/ozone/om/TestBucketOwner.java | 23 ++- .../hadoop/ozone/om/TestKeyPurging.java | 27 ++-- .../hadoop/ozone/om/TestListKeysWithFSO.java | 27 ++-- .../hadoop/ozone/om/TestListStatus.java | 27 ++-- .../ozone/om/TestOMEpochForNonRatis.java | 35 ++--- .../om/TestOMStartupWithBucketLayout.java | 20 +-- .../hadoop/ozone/om/TestObjectStore.java | 55 ++++---- .../ozone/om/TestObjectStoreWithFSO.java | 131 +++++++++--------- .../ozone/om/TestObjectStoreWithLegacyFS.java | 40 +++--- .../apache/hadoop/ozone/om/TestOmAcls.java | 30 ++-- .../ozone/om/TestOmBlockVersioning.java | 32 ++--- .../apache/hadoop/ozone/om/TestOmInit.java | 29 ++-- .../apache/hadoop/ozone/om/TestOmMetrics.java | 34 ++--- .../om/TestOzoneManagerConfiguration.java | 86 ++++++------ .../ozone/om/TestOzoneManagerListVolumes.java | 38 +++-- .../om/TestOzoneManagerRestInterface.java | 38 +++-- .../ozone/om/TestOzoneManagerRestart.java | 49 +++---- .../hadoop/ozone/om/TestScmSafeMode.java | 44 +++--- .../ozone/om/TestSecureOzoneManager.java | 84 ++++++----- .../om/multitenant/TestMultiTenantVolume.java | 44 +++--- .../TestOzoneManagerSnapshotProvider.java | 28 ++-- 22 files changed, 420 insertions(+), 535 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java index d54d5c420a1..69085830629 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java @@ -26,21 +26,21 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.junit.Assert; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + /** * Tests to verify bucket ops with older version client. */ +@Timeout(1200) public class TestBucketLayoutWithOlderClient { private static MiniOzoneCluster cluster = null; @@ -50,16 +50,13 @@ public class TestBucketLayoutWithOlderClient { private static String omId; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(1200000)); - /** * Create a MiniDFSCluster for testing. *

* * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -79,12 +76,12 @@ public void testCreateBucketWithOlderClient() throws Exception { OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, null); String volumeName = bucket.getVolumeName(); // OM defaulted bucket layout - Assert.assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); + assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); // Sets bucket layout explicitly. OzoneBucket fsobucket = TestDataUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, fsobucket.getBucketLayout()); // Create bucket request by an older client. @@ -111,19 +108,18 @@ public void testCreateBucketWithOlderClient() throws Exception { omResponse = cluster.getOzoneManager().getOmServerProtocol() .submitRequest(null, createBucketReq); - Assert.assertEquals(omResponse.getStatus(), - OzoneManagerProtocolProtos.Status.OK); + assertEquals(OzoneManagerProtocolProtos.Status.OK, omResponse.getStatus()); OmBucketInfo bucketInfo = cluster.getOzoneManager().getBucketInfo(volumeName, buckName); - Assert.assertNotNull(bucketInfo); - Assert.assertEquals(BucketLayout.LEGACY, bucketInfo.getBucketLayout()); + assertNotNull(bucketInfo); + assertEquals(BucketLayout.LEGACY, bucketInfo.getBucketLayout()); } /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java index 5b6d1d2f5a1..0af4925dbce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java @@ -31,14 +31,10 @@ import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,13 +47,14 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test for Ozone Bucket Owner. */ +@Timeout(120) public class TestBucketOwner { - @Rule public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(120)); private static MiniOzoneCluster cluster; private static final Logger LOG = @@ -71,7 +68,7 @@ public class TestBucketOwner { private static UserGroupInformation user3 = UserGroupInformation .createUserForTesting("user3", new String[] {"test3"}); - @BeforeClass + @BeforeAll public static void init() throws Exception { // loginUser is the user running this test. UserGroupInformation.setLoginUser(adminUser); @@ -103,7 +100,7 @@ public static void init() throws Exception { } } - @AfterClass + @AfterAll public static void stopCluster() { if (cluster != null) { cluster.shutdown(); @@ -244,7 +241,7 @@ private static void setVolumeAcl(ObjectStore store, String volumeName, String aclString) throws IOException { OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volumeName) .setResType(OzoneObj.ResourceType.VOLUME).setStoreType(OZONE).build(); - Assert.assertTrue(store.setAcl(obj, OzoneAcl.parseAcls(aclString))); + assertTrue(store.setAcl(obj, OzoneAcl.parseAcls(aclString))); } private void createKey(OzoneBucket ozoneBucket, String key, int length, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java index f79b6c1f7ff..066ff6e1db1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java @@ -33,14 +33,11 @@ import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.JUnit5AwareTimeout; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.ArrayList; @@ -48,6 +45,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertTrue; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; @@ -55,14 +53,9 @@ /** * Test OM's {@link KeyDeletingService}. */ +@Timeout(300) public class TestKeyPurging { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private MiniOzoneCluster cluster; private ObjectStore store; private OzoneManager om; @@ -71,7 +64,7 @@ public class TestKeyPurging { private static final int KEY_SIZE = 100; private OzoneClient client; - @Before + @BeforeEach public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, @@ -92,7 +85,7 @@ public void setup() throws Exception { om = cluster.getOzoneManager(); } - @After + @AfterEach public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -141,7 +134,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception { () -> keyDeletingService.getDeletedKeyCount().get() >= NUM_KEYS, 1000, 10000); - Assert.assertTrue(keyDeletingService.getRunCount().get() > 1); + assertTrue(keyDeletingService.getRunCount().get() > 1); GenericTestUtils.waitFor( () -> { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 5174fe5af77..d83bd487195 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -31,14 +31,10 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -52,11 +48,13 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * Test covers listKeys(keyPrefix, startKey) combinations * in a FSO bucket layout type. */ +@Timeout(1200) public class TestListKeysWithFSO { private static MiniOzoneCluster cluster = null; @@ -71,16 +69,13 @@ public class TestListKeysWithFSO { private static OzoneBucket fsoOzoneBucket2; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(1200000)); - /** * Create a MiniDFSCluster for testing. *

* * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, @@ -129,7 +124,7 @@ public static void init() throws Exception { initFSNameSpace(); } - @AfterClass + @AfterAll public static void teardownClass() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -587,7 +582,7 @@ private void checkKeyList(String keyPrefix, String startKey, List keyLists = new ArrayList<>(); while (ozoneKeyIterator.hasNext()) { OzoneKey ozoneKey = ozoneKeyIterator.next(); - Assert.assertEquals(expectedReplication, ozoneKey.getReplicationConfig()); + assertEquals(expectedReplication, ozoneKey.getReplicationConfig()); keyLists.add(ozoneKey.getName()); } LinkedList outputKeysList = new LinkedList(keyLists); @@ -598,7 +593,7 @@ private void checkKeyList(String keyPrefix, String startKey, } System.out.println("END:::keyPrefix---> " + keyPrefix + ":::---> " + startKey); - Assert.assertEquals(keys, outputKeysList); + assertEquals(keys, outputKeysList); } private void checkKeyList(String keyPrefix, String startKey, @@ -637,7 +632,7 @@ private static void createKey(OzoneBucket ozoneBucket, String key, int length, ozoneInputStream.read(read, 0, length); ozoneInputStream.close(); - Assert.assertEquals(new String(input, StandardCharsets.UTF_8), + assertEquals(new String(input, StandardCharsets.UTF_8), new String(read, StandardCharsets.UTF_8)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index a8ab1aca0e1..13e44402363 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -27,25 +27,23 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.AfterClass; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.UUID; import java.util.List; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.apache.hadoop.ozone.OzoneConfigKeys. OZONE_FS_ITERATE_BATCH_SIZE; /** * A simple test that asserts that list status output is sorted. */ +@Timeout(1200) public class TestListStatus { private static MiniOzoneCluster cluster = null; @@ -56,16 +54,13 @@ public class TestListStatus { private static OzoneBucket fsoOzoneBucket; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(1200000)); - /** * Create a MiniDFSCluster for testing. *

* * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, @@ -88,7 +83,7 @@ public static void init() throws Exception { buildNameSpaceTree(fsoOzoneBucket); } - @AfterClass + @AfterAll public static void teardownClass() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -193,7 +188,7 @@ private void checkKeyList(String keyPrefix, String startKey, List statuses = fsoOzoneBucket.listStatus(keyPrefix, false, startKey, numEntries, isPartialPrefix); - Assert.assertEquals(expectedNumKeys, statuses.size()); + assertEquals(expectedNumKeys, statuses.size()); System.out.println("BEGIN:::keyPrefix---> " + keyPrefix + ":::---> " + startKey); @@ -203,7 +198,7 @@ private void checkKeyList(String keyPrefix, String startKey, OzoneFileStatus stNext = statuses.get(i + 1); System.out.println("status:" + stCurr); - Assert.assertTrue(stCurr.getPath().compareTo(stNext.getPath()) < 0); + assertTrue(stCurr.getPath().compareTo(stNext.getPath()) < 0); } if (!statuses.isEmpty()) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMEpochForNonRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMEpochForNonRatis.java index 0291a4e5805..d63aab63dd4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMEpochForNonRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMEpochForNonRatis.java @@ -38,15 +38,12 @@ import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import static org.junit.jupiter.api.Assertions.assertEquals; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OmUtils.EPOCH_ID_SHIFT; import static org.apache.hadoop.ozone.OmUtils.EPOCH_WHEN_RATIS_NOT_ENABLED; @@ -55,6 +52,7 @@ /** * Tests OM epoch generation for when Ratis is not enabled. */ +@Timeout(240) public class TestOMEpochForNonRatis { private static MiniOzoneCluster cluster = null; private static OzoneConfiguration conf; @@ -63,10 +61,7 @@ public class TestOMEpochForNonRatis { private static String omId; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(240)); - - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -85,7 +80,7 @@ public static void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -121,8 +116,8 @@ public void testUniqueTrxnIndexOnOMRestart() throws Exception { OmVolumeArgs volumeInfo = omClient.getVolumeInfo(volumeName); long volumeTrxnIndex = OmUtils.getTxIdFromObjectId( volumeInfo.getObjectID()); - Assert.assertEquals(1, volumeTrxnIndex); - Assert.assertEquals(volumeTrxnIndex, om.getLastTrxnIndexForNonRatis()); + assertEquals(1, volumeTrxnIndex); + assertEquals(volumeTrxnIndex, om.getLastTrxnIndexForNonRatis()); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); ozoneVolume.createBucket(bucketName); @@ -131,8 +126,8 @@ public void testUniqueTrxnIndexOnOMRestart() throws Exception { OmBucketInfo bucketInfo = omClient.getBucketInfo(volumeName, bucketName); long bucketTrxnIndex = OmUtils.getTxIdFromObjectId( bucketInfo.getObjectID()); - Assert.assertEquals(2, bucketTrxnIndex); - Assert.assertEquals(bucketTrxnIndex, om.getLastTrxnIndexForNonRatis()); + assertEquals(2, bucketTrxnIndex); + assertEquals(bucketTrxnIndex, om.getLastTrxnIndexForNonRatis()); // Restart the OM and create new object cluster.restartOzoneManager(); @@ -154,10 +149,10 @@ public void testUniqueTrxnIndexOnOMRestart() throws Exception { .build()); long keyTrxnIndex = OmUtils.getTxIdFromObjectId( omKeyInfo.getObjectID()); - Assert.assertEquals(3, keyTrxnIndex); + assertEquals(3, keyTrxnIndex); // Key commit is a separate transaction. Hence, the last trxn index in DB // should be 1 more than KeyTrxnIndex - Assert.assertEquals(4, om.getLastTrxnIndexForNonRatis()); + assertEquals(4, om.getLastTrxnIndexForNonRatis()); } @Test @@ -179,6 +174,6 @@ public void testEpochIntegrationInObjectID() throws Exception { long volObjId = omClient.getVolumeInfo(volumeName).getObjectID(); long epochInVolObjId = volObjId >> EPOCH_ID_SHIFT; - Assert.assertEquals(EPOCH_WHEN_RATIS_NOT_ENABLED, epochInVolObjId); + assertEquals(EPOCH_WHEN_RATIS_NOT_ENABLED, epochInVolObjId); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java index 130c2fdce64..50cb298977a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java @@ -23,26 +23,18 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.UUID; /** * Verifies OM startup with different layout. */ +@Timeout(300) public class TestOMStartupWithBucketLayout { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private static MiniOzoneCluster cluster; private static OzoneClient client; @@ -161,8 +153,8 @@ public void testRestartWithOBSLayout() throws Exception { private void verifyBucketLayout(OzoneBucket bucket, BucketLayout metadataLayout) { - Assert.assertNotNull(bucket); - Assert.assertEquals(metadataLayout, bucket.getBucketLayout()); + Assertions.assertNotNull(bucket); + Assertions.assertEquals(metadataLayout, bucket.getBucketLayout()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java index bb4fafec278..5349727ff5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java @@ -26,21 +26,21 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + /** * Tests to verify Object store without prefix enabled. */ +@Timeout(1200) public class TestObjectStore { private static MiniOzoneCluster cluster = null; private static OzoneConfiguration conf; @@ -49,16 +49,13 @@ public class TestObjectStore { private static String omId; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(1200000)); - /** * Create a MiniOzoneCluster for testing. *

* * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -73,7 +70,7 @@ public static void init() throws Exception { /** * Shutdown MiniOzoneCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -93,8 +90,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { BucketArgs.Builder builder = BucketArgs.newBuilder(); volume.createBucket(sampleBucketName, builder.build()); OzoneBucket bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); // Case 2: Bucket layout: OBJECT_STORE @@ -102,8 +99,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder.setBucketLayout(BucketLayout.OBJECT_STORE); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.OBJECT_STORE, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); // Case 3: Bucket layout: LEGACY @@ -111,16 +108,16 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder.setBucketLayout(BucketLayout.LEGACY); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); // Case 3: Bucket layout: FILE_SYSTEM_OPTIMIZED sampleBucketName = UUID.randomUUID().toString(); builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); } @@ -164,16 +161,16 @@ public void testCreateLinkBucketWithBucketLayout() throws Exception { // Check that Link Buckets' layouts match source bucket layouts OzoneBucket bucket = volume.getBucket(linkBucket1Name); - Assert.assertEquals(sourceBucket1Layout, bucket.getBucketLayout()); + assertEquals(sourceBucket1Layout, bucket.getBucketLayout()); bucket = volume.getBucket(linkBucket2Name); - Assert.assertEquals(sourceBucket2Layout, bucket.getBucketLayout()); + assertEquals(sourceBucket2Layout, bucket.getBucketLayout()); // linkBucket3 is chained onto linkBucket1, hence its bucket layout matches // linkBucket1's source bucket. bucket = volume.getBucket(linkBucket3Name); - Assert.assertEquals(sourceBucket1Layout, bucket.getBucketLayout()); - Assert.assertEquals(linkBucket1Name, bucket.getSourceBucket()); + assertEquals(sourceBucket1Layout, bucket.getBucketLayout()); + assertEquals(linkBucket1Name, bucket.getSourceBucket()); } @Test @@ -198,10 +195,10 @@ public void testCreateDanglingLinkBucket() throws Exception { // since sourceBucket does not exist, layout depends on // OZONE_DEFAULT_BUCKET_LAYOUT config. OzoneBucket bucket = volume.getBucket(danglingLinkBucketName); - Assert.assertEquals(BucketLayout.fromString( + assertEquals(BucketLayout.fromString( conf.get(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT)), bucket.getBucketLayout()); - Assert.assertEquals(sourceBucketName, bucket.getSourceBucket()); + assertEquals(sourceBucketName, bucket.getSourceBucket()); } @Test @@ -225,10 +222,10 @@ public void testLoopInLinkBuckets() throws Exception { try { volume.getBucket(linkBucket1Name); - Assert.fail("Should throw Exception due to loop in Link Buckets"); + fail("Should throw Exception due to loop in Link Buckets"); } catch (OMException oe) { // Expected exception - Assert.assertEquals(OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS, + assertEquals(OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS, oe.getResult()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index d45ebe40827..c61dfd36a98 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -48,15 +48,11 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.util.StringUtils; import org.apache.ozone.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.AfterClass; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.net.URI; @@ -76,12 +72,17 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests to verify Object store with prefix enabled cases. */ +@Timeout(1200) public class TestObjectStoreWithFSO { private static final Path ROOT = new Path(OZONE_URI_DELIMITER); @@ -95,16 +96,13 @@ public class TestObjectStoreWithFSO { private static FileSystem fs; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(new Timeout(1200000)); - /** * Create a MiniDFSCluster for testing. *

* * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -132,7 +130,7 @@ public static void init() throws Exception { fs = FileSystem.get(conf); } - @After + @AfterEach public void tearDown() throws Exception { deleteRootDir(); } @@ -151,8 +149,7 @@ protected void deleteRootDir() throws IOException { deleteRootRecursively(fileStatuses); fileStatuses = fs.listStatus(ROOT); if (fileStatuses != null) { - Assert.assertEquals( - "Delete root failed!", 0, fileStatuses.length); + assertEquals(0, fileStatuses.length, "Delete root failed!"); } } @@ -171,9 +168,9 @@ public void testCreateKey() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); Table openFileTable = cluster.getOzoneManager().getMetadataManager() @@ -192,7 +189,7 @@ public void testCreateKey() throws Exception { long clientID = keyOutputStream.getClientID(); OmDirectoryInfo dirPathC = getDirInfo(parent); - Assert.assertNotNull("Failed to find dir path: a/b/c", dirPathC); + assertNotNull(dirPathC, "Failed to find dir path: a/b/c"); // after file creation verifyKeyInOpenFileTable(openFileTable, clientID, file, @@ -252,8 +249,8 @@ public void testDeleteBucketWithKeys() throws Exception { // Create a key. ozoneBucket.createKey(key, 10).close(); - Assert.assertFalse(cluster.getOzoneManager().getMetadataManager() - .isBucketEmpty(testVolumeName, testBucketName)); + assertFalse(cluster.getOzoneManager().getMetadataManager().isBucketEmpty( + testVolumeName, testBucketName)); try { // Try to delete the bucket while a key is present under it. @@ -265,7 +262,7 @@ public void testDeleteBucketWithKeys() throws Exception { // Delete the key (this only deletes the file) ozoneBucket.deleteKey(key); - Assert.assertFalse(cluster.getOzoneManager().getMetadataManager() + assertFalse(cluster.getOzoneManager().getMetadataManager() .isBucketEmpty(testVolumeName, testBucketName)); try { // Try to delete the bucket while intermediate dirs are present under it. @@ -278,7 +275,7 @@ public void testDeleteBucketWithKeys() throws Exception { // Delete last level of directories. ozoneBucket.deleteDirectory(parent, true); - Assert.assertFalse(cluster.getOzoneManager().getMetadataManager() + assertFalse(cluster.getOzoneManager().getMetadataManager() .isBucketEmpty(testVolumeName, testBucketName)); try { // Try to delete the bucket while dirs are present under it. @@ -291,7 +288,7 @@ public void testDeleteBucketWithKeys() throws Exception { // Delete all the intermediate directories ozoneBucket.deleteDirectory("a/", true); - Assert.assertTrue(cluster.getOzoneManager().getMetadataManager() + assertTrue(cluster.getOzoneManager().getMetadataManager() .isBucketEmpty(testVolumeName, testBucketName)); ozoneVolume.deleteBucket(testBucketName); // Cleanup the Volume. @@ -306,9 +303,9 @@ public void testLookupKey() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); Table openFileTable = cluster.getOzoneManager().getMetadataManager() @@ -324,7 +321,7 @@ public void testLookupKey() throws Exception { long clientID = keyOutputStream.getClientID(); OmDirectoryInfo dirPathC = getDirInfo(parent); - Assert.assertNotNull("Failed to find dir path: a/b/c", dirPathC); + assertNotNull(dirPathC, "Failed to find dir path: a/b/c"); // after file creation verifyKeyInOpenFileTable(openFileTable, clientID, fileName, @@ -346,7 +343,7 @@ public void testLookupKey() throws Exception { ozoneOutputStream.close(); OzoneKeyDetails keyDetails = ozoneBucket.getKey(key); - Assert.assertEquals(key, keyDetails.getName()); + assertEquals(key, keyDetails.getName()); Table fileTable = cluster.getOzoneManager().getMetadataManager() @@ -400,9 +397,9 @@ public void testLookupKey() throws Exception { public void testListKeysAtDifferentLevels() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); String keyc1 = "/a/b1/c1/c1.tx"; String keyc2 = "/a/b1/c2/c2.tx"; @@ -532,9 +529,9 @@ private void verifyFullTreeStructure(Iterator keyItr) { public void testListKeysWithNotNormalizedPath() throws Exception { ObjectStore objectStore = client.getObjectStore(); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); String key1 = "/dir1///dir2/file1/"; String key2 = "/dir1///dir2/file2/"; @@ -595,7 +592,7 @@ private void checkKeyList(Iterator ozoneKeyIterator, outputKeys.add(ozoneKey.getName()); } - Assert.assertEquals(keys, outputKeys); + assertEquals(keys, outputKeys); } private void createKeys(OzoneBucket ozoneBucket, List keys) @@ -625,7 +622,7 @@ private void createKey(OzoneBucket ozoneBucket, String key, int length, ozoneInputStream.close(); String inputString = new String(input, StandardCharsets.UTF_8); - Assert.assertEquals(inputString, new String(read, StandardCharsets.UTF_8)); + assertEquals(inputString, new String(read, StandardCharsets.UTF_8)); // Read using filesystem. String rootPath = String.format("%s://%s.%s/", OZONE_URI_SCHEME, @@ -637,7 +634,7 @@ private void createKey(OzoneBucket ozoneBucket, String key, int length, fsDataInputStream.read(read, 0, length); fsDataInputStream.close(); - Assert.assertEquals(inputString, new String(read, StandardCharsets.UTF_8)); + assertEquals(inputString, new String(read, StandardCharsets.UTF_8)); } @Test @@ -655,7 +652,7 @@ public void testRenameKey() throws IOException { String toKeyName = ""; bucket.renameKey(fromKeyName, toKeyName); OzoneKey emptyKeyRename = bucket.getKey(fromKeyName); - Assert.assertEquals(fromKeyName, emptyKeyRename.getName()); + assertEquals(fromKeyName, emptyKeyRename.getName()); toKeyName = UUID.randomUUID().toString(); bucket.renameKey(fromKeyName, toKeyName); @@ -665,11 +662,11 @@ public void testRenameKey() throws IOException { bucket.getKey(fromKeyName); fail("Lookup for old from key name should fail!"); } catch (OMException ome) { - Assert.assertEquals(KEY_NOT_FOUND, ome.getResult()); + assertEquals(KEY_NOT_FOUND, ome.getResult()); } OzoneKey key = bucket.getKey(toKeyName); - Assert.assertEquals(toKeyName, key.getName()); + assertEquals(toKeyName, key.getName()); } @Test @@ -691,8 +688,8 @@ public void testKeyRenameWithSubDirs() throws Exception { bucket.renameKey(keyName2, newKeyName2); // new key should exist - Assert.assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName()); - Assert.assertEquals(newKeyName2, bucket.getKey(newKeyName2).getName()); + assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName()); + assertEquals(newKeyName2, bucket.getKey(newKeyName2).getName()); // old key should not exist assertKeyRenamedEx(bucket, keyName1); @@ -715,7 +712,7 @@ public void testRenameToAnExistingKey() throws Exception { bucket.renameKey(keyName1, keyName2); fail("Should throw exception as destin key already exists!"); } catch (OMException e) { - Assert.assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); } } @@ -732,8 +729,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); volume.createBucket(sampleBucketName, builder.build()); OzoneBucket bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); // Case 2: Bucket layout: OBJECT_STORE @@ -741,8 +738,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder.setBucketLayout(BucketLayout.OBJECT_STORE); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout()); // Case 3: Bucket layout: Empty and // OM default bucket layout: FILE_SYSTEM_OPTIMIZED @@ -750,8 +747,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { sampleBucketName = UUID.randomUUID().toString(); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); // Case 4: Bucket layout: Empty @@ -759,8 +756,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder = BucketArgs.newBuilder(); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); // Case 5: Bucket layout: LEGACY @@ -768,8 +765,8 @@ public void testCreateBucketWithBucketLayout() throws Exception { builder.setBucketLayout(BucketLayout.LEGACY); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); - Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); + assertEquals(sampleBucketName, bucket.getName()); + assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); } private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) @@ -778,7 +775,7 @@ private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) bucket.getKey(keyName); fail("Should throw KeyNotFound as the key got renamed!"); } catch (OMException ome) { - Assert.assertEquals(KEY_NOT_FOUND, ome.getResult()); + assertEquals(KEY_NOT_FOUND, ome.getResult()); } } @@ -790,7 +787,7 @@ private void createTestKey(OzoneBucket bucket, String keyName, out.write(keyValue.getBytes(StandardCharsets.UTF_8)); out.close(); OzoneKey key = bucket.getKey(keyName); - Assert.assertEquals(keyName, key.getName()); + assertEquals(keyName, key.getName()); } private OmDirectoryInfo getDirInfo(String parentKey) throws Exception { @@ -822,15 +819,14 @@ private void verifyKeyInFileTable(Table fileTable, parentID, fileName); OmKeyInfo omKeyInfo = fileTable.get(dbFileKey); if (isEmpty) { - Assert.assertNull("Table is not empty!", omKeyInfo); + assertNull(omKeyInfo, "Table is not empty!"); } else { - Assert.assertNotNull("Table is empty!", omKeyInfo); + assertNotNull(omKeyInfo, "Table is empty!"); // used startsWith because the key format is, // /fileName/ and clientID is not visible. - Assert.assertEquals("Invalid Key: " + omKeyInfo.getObjectInfo(), - omKeyInfo.getKeyName(), fileName); - Assert.assertEquals("Invalid Key", parentID, - omKeyInfo.getParentObjectID()); + assertEquals(omKeyInfo.getKeyName(), fileName, + "Invalid Key: " + omKeyInfo.getObjectInfo()); + assertEquals(parentID, omKeyInfo.getParentObjectID(), "Invalid Key"); } } @@ -851,20 +847,19 @@ private void verifyKeyInOpenFileTable(Table openFileTable, OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); return omKeyInfo == null; } catch (IOException e) { - Assert.fail("DB failure!"); + fail("DB failure!"); return false; } }, 1000, 120000); } else { OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); - Assert.assertNotNull("Table is empty!", omKeyInfo); + assertNotNull(omKeyInfo, "Table is empty!"); // used startsWith because the key format is, // /fileName/ and clientID is not visible. - Assert.assertEquals("Invalid Key: " + omKeyInfo.getObjectInfo(), - omKeyInfo.getKeyName(), fileName); - Assert.assertEquals("Invalid Key", parentID, - omKeyInfo.getParentObjectID()); + assertEquals(omKeyInfo.getKeyName(), fileName, + "Invalid Key: " + omKeyInfo.getObjectInfo()); + assertEquals(parentID, omKeyInfo.getParentObjectID(), "Invalid Key"); } } @@ -875,7 +870,7 @@ public BucketLayout getBucketLayout() { /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java index ad1e5ee43df..0b00f9b5780 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java @@ -43,15 +43,11 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.ozone.test.GenericTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,14 +57,16 @@ import java.util.Map; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + /** * Test verifies object store with OZONE_OM_ENABLE_FILESYSTEM_PATHS enabled. */ +@Timeout(200) public class TestObjectStoreWithLegacyFS { - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(200)); - private static MiniOzoneCluster cluster = null; private static OzoneClient client; @@ -81,7 +79,7 @@ public class TestObjectStoreWithLegacyFS { private static final Logger LOG = LoggerFactory.getLogger(TestObjectStoreWithLegacyFS.class); - @BeforeClass + @BeforeAll public static void initClass() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); @@ -98,7 +96,7 @@ public static void initClass() throws Exception { /** * Shutdown MiniOzoneCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -106,7 +104,7 @@ public static void shutdown() { } } - @Before + @BeforeEach public void init() throws Exception { volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); @@ -167,10 +165,10 @@ private boolean assertKeyCount( } } catch (IOException ex) { LOG.info("Test failed with: " + ex.getMessage(), ex); - Assert.fail("Test failed with: " + ex.getMessage()); + fail("Test failed with: " + ex.getMessage()); } if (countKeys > expectedCnt) { - Assert.fail("Test failed with: too many keys found, expected " + fail("Test failed with: too many keys found, expected " + expectedCnt + " keys, found " + countKeys + " keys"); } if (matchingKeys != expectedCnt) { @@ -196,7 +194,7 @@ public void testMultiPartCompleteUpload() throws Exception { omMultipartUploadCompleteInfo = uploadMPUWithDirectoryExists(bucket, keyName); // successfully uploaded MPU key - Assert.assertNotNull(omMultipartUploadCompleteInfo); + assertNotNull(omMultipartUploadCompleteInfo); // Test-2: Upload MPU to an LEGACY layout with Directory Exists legacyBuckName = UUID.randomUUID().toString(); @@ -209,10 +207,10 @@ public void testMultiPartCompleteUpload() throws Exception { try { uploadMPUWithDirectoryExists(bucket, keyName); - Assert.fail("Must throw error as there is " + + fail("Must throw error as there is " + "already directory in the given path"); } catch (OMException ome) { - Assert.assertEquals(OMException.ResultCodes.NOT_A_FILE, ome.getResult()); + assertEquals(OMException.ResultCodes.NOT_A_FILE, ome.getResult()); } } @@ -221,7 +219,7 @@ private OmMultipartUploadCompleteInfo uploadMPUWithDirectoryExists( OmMultipartInfo omMultipartInfo = bucket.initiateMultipartUpload(keyName, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); - Assert.assertNotNull(omMultipartInfo.getUploadID()); + assertNotNull(omMultipartInfo.getUploadID()); String uploadID = omMultipartInfo.getUploadID(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java index 003fb448890..d0d2bf40efd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java @@ -33,14 +33,11 @@ import org.apache.hadoop.ozone.security.acl.RequestContext; import org.apache.hadoop.ozone.audit.AuditLogTestUtils; import org.apache.ozone.test.GenericTestUtils; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.ArrayList; @@ -51,21 +48,16 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.audit.AuditLogTestUtils.verifyAuditLog; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test for Ozone Manager ACLs. */ +@Timeout(300) public class TestOmAcls { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private static boolean volumeAclAllow = true; private static boolean bucketAclAllow = true; private static boolean keyAclAllow = true; @@ -83,7 +75,7 @@ public class TestOmAcls { *

* Ozone is made active by setting OZONE_ENABLED = true */ - @BeforeClass + @BeforeAll public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String clusterId = UUID.randomUUID().toString(); @@ -104,7 +96,7 @@ public static void init() throws Exception { GenericTestUtils.LogCapturer.captureLogs(OzoneManager.getLogger()); } - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -113,7 +105,7 @@ public static void shutdown() { AuditLogTestUtils.deleteAuditLogFile(); } - @Before + @BeforeEach public void setup() throws IOException { logCapturer.clearOutput(); AuditLogTestUtils.truncateAuditLogFile(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java index dd39f933882..428bfa73059 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java @@ -36,38 +36,28 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.commons.lang3.RandomStringUtils; -import org.junit.AfterClass; -import org.junit.Assert; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.junit.Assert.assertEquals; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; /** * This class tests the versioning of blocks from OM side. */ +@Timeout(300) public class TestOmBlockVersioning { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); private static MiniOzoneCluster cluster = null; private static OzoneClient client; private static OzoneConfiguration conf; private static OzoneManager ozoneManager; private static OzoneManagerProtocol writeClient; - @Rule - public ExpectedException exception = ExpectedException.none(); - /** * Create a MiniDFSCluster for testing. *

@@ -75,7 +65,7 @@ public class TestOmBlockVersioning { * * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); cluster = MiniOzoneCluster.newBuilder(conf).build(); @@ -89,7 +79,7 @@ public static void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -154,7 +144,7 @@ public void testAllocateCommit() throws Exception { List locationInfoList = openKey.getKeyInfo().getLatestVersionLocations() .getBlocksLatestVersionOnly(); - Assert.assertTrue(locationInfoList.size() == 1); + assertTrue(locationInfoList.size() == 1); locationInfoList.add(locationInfo); keyArgs.setLocationInfoList(locationInfoList); writeClient.commitKey(keyArgs, openKey.getId()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java index ec8a39d63db..b9feec93a90 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java @@ -23,26 +23,18 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; /** * Test Ozone Manager Init. */ +@Timeout(300) public class TestOmInit { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); private static MiniOzoneCluster cluster = null; private static OMMetrics omMetrics; private static OzoneConfiguration conf; @@ -50,9 +42,6 @@ public class TestOmInit { private static String scmId; private static String omId; - @Rule - public ExpectedException exception = ExpectedException.none(); - /** * Create a MiniDFSCluster for testing. *

@@ -60,7 +49,7 @@ public class TestOmInit { * * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -78,7 +67,7 @@ public static void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { if (cluster != null) { cluster.shutdown(); @@ -96,7 +85,7 @@ public void testOmInitAgain() throws IOException, // Stop the Ozone Manager cluster.getOzoneManager().stop(); // Now try to init the OM again. It should succeed - Assert.assertTrue(OzoneManager.omInit(conf)); + Assertions.assertTrue(OzoneManager.omInit(conf)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 7fc050e1722..d49f059a06c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -22,8 +22,9 @@ import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; @@ -66,26 +67,17 @@ import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.assertj.core.util.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.mockito.Mockito; /** * Test for OM metrics. */ +@Timeout(300) public class TestOmMetrics { - - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); private MiniOzoneCluster cluster; private MiniOzoneCluster.Builder clusterBuilder; private OzoneConfiguration conf; @@ -101,7 +93,7 @@ public class TestOmMetrics { /** * Create a MiniDFSCluster for testing. */ - @Before + @BeforeEach public void setup() throws Exception { conf = new OzoneConfiguration(); conf.setTimeDuration(OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL, @@ -121,7 +113,7 @@ private void startCluster() throws Exception { /** * Shutdown MiniDFSCluster. */ - @After + @AfterEach public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -129,8 +121,6 @@ public void shutdown() { } } - - @Test public void testVolumeOps() throws Exception { startCluster(); @@ -334,8 +324,8 @@ public void testKeyOps() throws Exception { writeClient.commitKey(keyArgs, keySession.getId()); } catch (Exception e) { //Expected Failure in preExecute due to not enough datanode - Assertions.assertTrue(e.getMessage() - .contains("No enough datanodes to choose"), e::getMessage); + assertTrue(e.getMessage().contains("No enough datanodes to choose"), + e::getMessage); } omMetrics = getMetrics("OMMetrics"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java index 62d50e5a701..8f75e568057 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java @@ -39,26 +39,22 @@ import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.util.LifeCycle; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests OM related configurations. */ +@Timeout(300) public class TestOzoneManagerConfiguration { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private OzoneConfiguration conf; private MiniOzoneCluster cluster; private String omId; @@ -69,7 +65,7 @@ public class TestOzoneManagerConfiguration { private static final long RATIS_RPC_TIMEOUT = 500L; - @Before + @BeforeEach public void init() throws IOException { conf = new OzoneConfiguration(); omId = UUID.randomUUID().toString(); @@ -87,7 +83,7 @@ public void init() throws IOException { omStore.initialize(); } - @After + @AfterEach public void shutdown() { if (cluster != null) { cluster.shutdown(); @@ -113,7 +109,7 @@ public void testNoConfiguredOMAddress() throws Exception { startCluster(); om = cluster.getOzoneManager(); - Assert.assertTrue(NetUtils.isLocalAddress( + assertTrue(NetUtils.isLocalAddress( om.getOmRpcServerAddr().getAddress())); } @@ -144,16 +140,16 @@ public void testDefaultPortIfNotSpecified() throws Exception { startCluster(); om = cluster.getOzoneManager(); - Assert.assertEquals("0.0.0.0", + assertEquals("0.0.0.0", om.getOmRpcServerAddr().getHostName()); - Assert.assertEquals(OMConfigKeys.OZONE_OM_PORT_DEFAULT, + assertEquals(OMConfigKeys.OZONE_OM_PORT_DEFAULT, om.getOmRpcServerAddr().getPort()); // Verify that the 2nd OMs address stored in the current OM also has the // default port as the port is not specified InetSocketAddress omNode2Addr = om.getPeerNodes().get(0).getRpcAddress(); - Assert.assertEquals("122.0.0.122", omNode2Addr.getHostString()); - Assert.assertEquals(OMConfigKeys.OZONE_OM_PORT_DEFAULT, + assertEquals("122.0.0.122", omNode2Addr.getHostString()); + assertEquals(OMConfigKeys.OZONE_OM_PORT_DEFAULT, omNode2Addr.getPort()); } @@ -169,14 +165,14 @@ public void testSingleNodeOMservice() throws Exception { om = cluster.getOzoneManager(); omRatisServer = om.getOmRatisServer(); - Assert.assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); + assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); // OM's Ratis server should have only 1 peer (itself) in its RaftGroup Collection peers = omRatisServer.getRaftGroup().getPeers(); - Assert.assertEquals(1, peers.size()); + assertEquals(1, peers.size()); // The RaftPeer id should match OM_DEFAULT_NODE_ID RaftPeer raftPeer = peers.toArray(new RaftPeer[1])[0]; - Assert.assertEquals(OzoneConsts.OM_DEFAULT_NODE_ID, + assertEquals(OzoneConsts.OM_DEFAULT_NODE_ID, raftPeer.getId().toString()); } @@ -220,15 +216,15 @@ public void testThreeNodeOMservice() throws Exception { om = cluster.getOzoneManager(); omRatisServer = om.getOmRatisServer(); - Assert.assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); + assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); // OM's Ratis server should have 3 peers in its RaftGroup Collection peers = omRatisServer.getRaftGroup().getPeers(); - Assert.assertEquals(3, peers.size()); + assertEquals(3, peers.size()); // Ratis server RaftPeerId should match with omNode2 ID as node2 is the // localhost - Assert.assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); + assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); // Verify peer details for (RaftPeer peer : peers) { @@ -247,9 +243,9 @@ public void testThreeNodeOMservice() throws Exception { // Ratis port is not set for node3. So it should take the default port expectedPeerAddress = "124.0.0.124:9898"; break; - default : Assert.fail("Unrecognized RaftPeerId"); + default : fail("Unrecognized RaftPeerId"); } - Assert.assertEquals(expectedPeerAddress, peer.getAddress()); + assertEquals(expectedPeerAddress, peer.getAddress()); } } @@ -298,19 +294,19 @@ public void testOMHAWithUnresolvedAddresses() throws Exception { // Verify Peer details List peerNodes = om.getPeerNodes(); for (OMNodeDetails peerNode : peerNodes) { - Assert.assertTrue(peerNode.isHostUnresolved()); - Assert.assertNull(peerNode.getInetAddress()); + assertTrue(peerNode.isHostUnresolved()); + assertNull(peerNode.getInetAddress()); } - Assert.assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); + assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); // OM's Ratis server should have 3 peers in its RaftGroup Collection peers = omRatisServer.getRaftGroup().getPeers(); - Assert.assertEquals(3, peers.size()); + assertEquals(3, peers.size()); // Ratis server RaftPeerId should match with omNode2 ID as node2 is the // localhost - Assert.assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); + assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); // Verify peer details for (RaftPeer peer : peers) { @@ -330,9 +326,9 @@ public void testOMHAWithUnresolvedAddresses() throws Exception { // Ratis port is not set for node3. So it should take the default port expectedPeerAddress = node3Hostname + ":9898"; break; - default : Assert.fail("Unrecognized RaftPeerId"); + default : fail("Unrecognized RaftPeerId"); } - Assert.assertEquals(expectedPeerAddress, peer.getAddress()); + assertEquals(expectedPeerAddress, peer.getAddress()); } } @@ -366,7 +362,7 @@ public void testWrongConfiguration() throws Exception { try { startCluster(); - Assert.fail("Wrong Configuration. OM initialization should have failed."); + fail("Wrong Configuration. OM initialization should have failed."); } catch (OzoneIllegalArgumentException e) { GenericTestUtils.assertExceptionContains("Configuration has no " + OMConfigKeys.OZONE_OM_ADDRESS_KEY + " address that matches local " + @@ -387,10 +383,10 @@ public void testNoOMNodes() throws Exception { try { startCluster(); - Assert.fail("Should have failed to start the cluster!"); + fail("Should have failed to start the cluster!"); } catch (OzoneIllegalArgumentException e) { // Expect error message - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( "List of OM Node ID's should be specified")); } } @@ -417,10 +413,10 @@ public void testNoOMAddrs() throws Exception { try { startCluster(); - Assert.fail("Should have failed to start the cluster!"); + fail("Should have failed to start the cluster!"); } catch (OzoneIllegalArgumentException e) { // Expect error message - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( "OM RPC Address should be set for all node")); } } @@ -470,16 +466,16 @@ public void testMultipleOMServiceIds() throws Exception { om = cluster.getOzoneManager(); omRatisServer = om.getOmRatisServer(); - Assert.assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); + assertEquals(LifeCycle.State.RUNNING, om.getOmRatisServerState()); // OM's Ratis server should have 3 peers in its RaftGroup Collection peers = omRatisServer.getRaftGroup().getPeers(); - Assert.assertEquals(3, peers.size()); + assertEquals(3, peers.size()); // Verify that the serviceId and nodeId match the node with the localhost // address - om-service-test2 and omNode2 - Assert.assertEquals(om2ServiceId, om.getOMServiceId()); - Assert.assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); + assertEquals(om2ServiceId, om.getOMServiceId()); + assertEquals(omNode2Id, omRatisServer.getRaftPeerId().toString()); } private String getOMAddrKeyWithSuffix(String serviceId, String nodeId) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java index c90f95f71b4..530da077c31 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java @@ -47,27 +47,22 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED; import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; /** * Test OzoneManager list volume operation under combinations of configs. */ +@Timeout(120) public class TestOzoneManagerListVolumes { private static MiniOzoneCluster cluster; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(120)); - private static UserGroupInformation adminUser = UserGroupInformation.createUserForTesting("om", new String[]{"ozone"}); private static UserGroupInformation user1 = @@ -80,7 +75,7 @@ public class TestOzoneManagerListVolumes { UserGroupInformation.createUserForTesting("user3@example.com", new String[]{"test"}); - @Before + @BeforeEach public void init() throws Exception { // loginUser is the user running this test. // Implication: loginUser is automatically added to the OM admin list. @@ -91,7 +86,7 @@ public void init() throws Exception { * Create a MiniDFSCluster for testing. */ - @BeforeClass + @BeforeAll public static void setupClass() throws InterruptedException, TimeoutException, IOException { OzoneConfiguration conf = new OzoneConfiguration(); @@ -131,7 +126,7 @@ public static void setupClass() om.join(); } - @AfterClass + @AfterAll public static void shutdownClass() { if (cluster != null) { cluster.shutdown(); @@ -148,7 +143,7 @@ private void startOM(boolean aclEnabled, cluster.getOzoneManager().restart(); } - @After + @AfterEach public void stopOM() { OzoneManager om = cluster.getOzoneManager(); if (om != null) { @@ -179,7 +174,8 @@ private static void setVolumeAcl(ObjectStore objectStore, String volumeName, String aclString) throws IOException { OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volumeName) .setResType(OzoneObj.ResourceType.VOLUME).setStoreType(OZONE).build(); - Assert.assertTrue(objectStore.setAcl(obj, OzoneAcl.parseAcls(aclString))); + Assertions.assertTrue(objectStore.setAcl( + obj, OzoneAcl.parseAcls(aclString))); } /** @@ -212,7 +208,7 @@ private static void checkUser(OzoneClient client, UserGroupInformation user, String volumeName = vol.getName(); accessibleVolumes.add(volumeName); } - Assert.assertEquals(new HashSet<>(expectVol), accessibleVolumes); + Assertions.assertEquals(new HashSet<>(expectVol), accessibleVolumes); } catch (RuntimeException ex) { if (expectListByUserSuccess) { throw ex; @@ -238,11 +234,11 @@ private static void checkUser(OzoneClient client, UserGroupInformation user, it.next(); count++; } - Assert.assertEquals(5, count); + Assertions.assertEquals(5, count); } else { try { objectStore.listVolumes("volume"); - Assert.fail("listAllVolumes should fail for " + user.getUserName()); + Assertions.fail("listAllVolumes should fail for " + user.getUserName()); } catch (RuntimeException ex) { // Current listAllVolumes throws RuntimeException if (ex.getCause() instanceof OMException) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java index 0a844c86487..2cca0619afe 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java @@ -32,45 +32,39 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; import java.net.InetSocketAddress; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; import static org.apache.hadoop.ozone.OmUtils.getOmAddressForClients; /** * This class is to test the REST interface exposed by OzoneManager. */ +@Timeout(300) public class TestOzoneManagerRestInterface { - /** - * Set a timeout for each test. - */ - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - private static MiniOzoneCluster cluster; private static OzoneConfiguration conf; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { conf = new OzoneConfiguration(); cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster.waitForClusterToBeReady(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (cluster != null) { cluster.shutdown(); @@ -102,19 +96,19 @@ public void testGetServiceList() throws Exception { getOmAddressForClients(conf); ServiceInfo omInfo = serviceMap.get(HddsProtos.NodeType.OM); - Assert.assertEquals(omAddress.getHostName(), omInfo.getHostname()); - Assert.assertEquals(omAddress.getPort(), + assertEquals(omAddress.getHostName(), omInfo.getHostname()); + assertEquals(omAddress.getPort(), omInfo.getPort(ServicePort.Type.RPC)); - Assert.assertEquals(server.getHttpAddress().getPort(), + assertEquals(server.getHttpAddress().getPort(), omInfo.getPort(ServicePort.Type.HTTP)); - Assert.assertTrue(getScmAddressForClients(conf).iterator().hasNext()); + assertTrue(getScmAddressForClients(conf).iterator().hasNext()); InetSocketAddress scmAddress = getScmAddressForClients(conf).iterator().next(); ServiceInfo scmInfo = serviceMap.get(HddsProtos.NodeType.SCM); - Assert.assertEquals(scmAddress.getHostName(), scmInfo.getHostname()); - Assert.assertEquals(scmAddress.getPort(), + assertEquals(scmAddress.getHostName(), scmInfo.getHostname()); + assertEquals(scmAddress.getPort(), scmInfo.getPort(ServicePort.Type.RPC)); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java index 5f6954e7907..c95bb1e35fa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java @@ -39,29 +39,27 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import org.junit.AfterClass; -import org.junit.Assert; - import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; -import static org.junit.Assert.fail; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test some client operations after cluster starts. And perform restart and * then performs client operations and check the behavior is expected or not. */ +@Timeout(240) public class TestOzoneManagerRestart { private static MiniOzoneCluster cluster = null; private static OzoneConfiguration conf; @@ -70,9 +68,6 @@ public class TestOzoneManagerRestart { private static String omId; private static OzoneClient client; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(240)); - /** * Create a MiniDFSCluster for testing. *

@@ -80,7 +75,7 @@ public class TestOzoneManagerRestart { * * @throws IOException */ - @BeforeClass + @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -104,7 +99,7 @@ public static void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @AfterClass + @AfterAll public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -121,7 +116,7 @@ public void testRestartOMWithVolumeOperation() throws Exception { objectStore.createVolume(volumeName); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); cluster.restartOzoneManager(); cluster.restartStorageContainerManager(true); @@ -136,7 +131,7 @@ public void testRestartOMWithVolumeOperation() throws Exception { // Get Volume. ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); } @@ -151,12 +146,12 @@ public void testRestartOMWithBucketOperation() throws Exception { objectStore.createVolume(volumeName); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); ozoneVolume.createBucket(bucketName); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); cluster.restartOzoneManager(); cluster.restartStorageContainerManager(true); @@ -171,7 +166,7 @@ public void testRestartOMWithBucketOperation() throws Exception { // Get bucket. ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); } @@ -191,12 +186,12 @@ public void testRestartOMWithKeyOperation() throws Exception { objectStore.createVolume(volumeName); OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); - Assert.assertTrue(ozoneVolume.getName().equals(volumeName)); + assertTrue(ozoneVolume.getName().equals(volumeName)); ozoneVolume.createBucket(bucketName); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - Assert.assertTrue(ozoneBucket.getName().equals(bucketName)); + assertTrue(ozoneBucket.getName().equals(bucketName)); String data = "random data"; OzoneOutputStream ozoneOutputStream1 = ozoneBucket.createKey(key1, @@ -213,14 +208,14 @@ public void testRestartOMWithKeyOperation() throws Exception { try { ozoneBucket.renameKeys(keyMap); } catch (OMException ex) { - Assert.assertEquals(PARTIAL_RENAME, ex.getResult()); + assertEquals(PARTIAL_RENAME, ex.getResult()); } // Get original Key1, it should not exist try { ozoneBucket.getKey(key1); } catch (OMException ex) { - Assert.assertEquals(KEY_NOT_FOUND, ex.getResult()); + assertEquals(KEY_NOT_FOUND, ex.getResult()); } cluster.restartOzoneManager(); @@ -232,15 +227,15 @@ public void testRestartOMWithKeyOperation() throws Exception { // Get newKey1. OzoneKey ozoneKey = ozoneBucket.getKey(newKey1); - Assert.assertTrue(ozoneKey.getName().equals(newKey1)); - Assert.assertTrue(ozoneKey.getReplicationType().equals( + assertTrue(ozoneKey.getName().equals(newKey1)); + assertTrue(ozoneKey.getReplicationType().equals( ReplicationType.RATIS)); // Get newKey2, it should not exist try { ozoneBucket.getKey(newKey2); } catch (OMException ex) { - Assert.assertEquals(KEY_NOT_FOUND, ex.getResult()); + assertEquals(KEY_NOT_FOUND, ex.getResult()); } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java index 7a70a6ddc2e..1d3aaf351fc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java @@ -43,17 +43,14 @@ import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.JUnit5AwareTimeout; import org.apache.ozone.test.UnhealthyTest; import org.apache.ozone.test.tag.Unhealthy; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,15 +63,16 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test Ozone Manager operation in distributed handler scenario. */ +@Timeout(300) @Category(UnhealthyTest.class) @Unhealthy("HDDS-3260") public class TestScmSafeMode { @@ -88,10 +86,6 @@ public class TestScmSafeMode { private StorageContainerLocationProtocolClientSideTranslatorPB storageContainerLocationClient; - - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); - /** * Create a MiniDFSCluster for testing. *

@@ -100,7 +94,7 @@ public class TestScmSafeMode { * * @throws IOException */ - @Before + @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); conf.set(OZONE_SCM_STALENODE_INTERVAL, "10s"); @@ -121,7 +115,7 @@ public void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @After + @AfterEach public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -166,7 +160,7 @@ public void testSafeModeOperations() throws Exception { StorageContainerManager scm; scm = cluster.getStorageContainerManager(); - Assert.assertTrue(scm.isInSafeMode()); + Assertions.assertTrue(scm.isInSafeMode()); om = cluster.getOzoneManager(); @@ -189,20 +183,20 @@ public void testSafeModeOperations() throws Exception { @Test public void testIsScmInSafeModeAndForceExit() throws Exception { // Test 1: SCM should be out of safe mode. - Assert.assertFalse(storageContainerLocationClient.inSafeMode()); + Assertions.assertFalse(storageContainerLocationClient.inSafeMode()); cluster.stop(); // Restart the cluster with same metadata dir. try { cluster = builder.build(); } catch (IOException e) { - Assert.fail("Cluster startup failed."); + Assertions.fail("Cluster startup failed."); } // Test 2: Scm should be in safe mode as datanodes are not started yet. storageContainerLocationClient = cluster .getStorageContainerLocationClient(); - Assert.assertTrue(storageContainerLocationClient.inSafeMode()); + Assertions.assertTrue(storageContainerLocationClient.inSafeMode()); // Force scm out of safe mode. cluster.getStorageContainerManager().getClientProtocolServer() .forceExitSafeMode(); @@ -212,7 +206,7 @@ public void testIsScmInSafeModeAndForceExit() throws Exception { return !cluster.getStorageContainerManager().getClientProtocolServer() .inSafeMode(); } catch (IOException e) { - Assert.fail("Cluster"); + Assertions.fail("Cluster"); return false; } }, 10, 1000 * 5); @@ -227,7 +221,7 @@ public void testSCMSafeMode() throws Exception { try { cluster = builder.build(); } catch (IOException e) { - Assert.fail("Cluster startup failed."); + Assertions.fail("Cluster startup failed."); } assertTrue(cluster.getStorageContainerManager().isInSafeMode()); cluster.startHddsDatanodes(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java index 38d4ca1cff1..8562a09be08 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java @@ -29,14 +29,10 @@ import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.bouncycastle.cert.X509CertificateHolder; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.nio.file.Path; import java.nio.file.Paths; @@ -55,10 +51,15 @@ import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.apache.ozone.test.GenericTestUtils.LogCapturer; import static org.apache.ozone.test.GenericTestUtils.getTempPath; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Test secure Ozone Manager operation in distributed handler scenario. */ +@Timeout(25) public class TestSecureOzoneManager { private static final String COMPONENT = "om"; @@ -70,15 +71,12 @@ public class TestSecureOzoneManager { private Path metaDir; private HddsProtos.OzoneManagerDetailsProto omInfo; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(25)); - /** * Create a MiniDFSCluster for testing. *

* Ozone is made active by setting OZONE_ENABLED = true */ - @Before + @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -99,7 +97,7 @@ public void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @After + @AfterEach public void shutdown() { if (cluster != null) { cluster.shutdown(); @@ -127,22 +125,22 @@ public void testSecureOmInitFailures() throws Exception { OMCertificateClient client = new OMCertificateClient( securityConfig, null, omStorage, omInfo, "", scmId, null, null); - Assert.assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); + assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); privateKey = client.getPrivateKey(); publicKey = client.getPublicKey(); - Assert.assertNotNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNull(client.getCertificate()); + assertNotNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNull(client.getCertificate()); client.close(); // Case 2: If key pair already exist than response should be GETCERT. client = new OMCertificateClient( securityConfig, null, omStorage, omInfo, "", scmId, null, null); - Assert.assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); - Assert.assertNotNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); + assertNotNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNull(client.getCertificate()); client.close(); // Case 3: When public key as well as certificate is missing. @@ -151,10 +149,10 @@ public void testSecureOmInitFailures() throws Exception { securityConfig, null, omStorage, omInfo, "", null, null, null); FileUtils.deleteQuietly(Paths.get(securityConfig.getKeyLocation(COMPONENT) .toString(), securityConfig.getPublicKeyFileName()).toFile()); - Assert.assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); - Assert.assertNotNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); + assertNotNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNull(client.getCertificate()); client.close(); // Case 4: When private key and certificate is missing. @@ -164,10 +162,10 @@ public void testSecureOmInitFailures() throws Exception { KeyCodec keyCodec = new KeyCodec(securityConfig, COMPONENT); FileUtils.deleteQuietly(Paths.get(securityConfig.getKeyLocation(COMPONENT) .toString(), securityConfig.getPrivateKeyFileName()).toFile()); - Assert.assertEquals(CertificateClient.InitResponse.FAILURE, client.init()); - Assert.assertNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.FAILURE, client.init()); + assertNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNull(client.getCertificate()); client.close(); // Case 5: When only certificate is present. @@ -184,10 +182,10 @@ public void testSecureOmInitFailures() throws Exception { client = new OMCertificateClient( securityConfig, null, omStorage, omInfo, "", scmId, null, null); - Assert.assertEquals(CertificateClient.InitResponse.FAILURE, client.init()); - Assert.assertNull(client.getPrivateKey()); - Assert.assertNull(client.getPublicKey()); - Assert.assertNotNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.FAILURE, client.init()); + assertNull(client.getPrivateKey()); + assertNull(client.getPublicKey()); + assertNotNull(client.getCertificate()); client.close(); // Case 6: When private key and certificate is present. @@ -197,20 +195,20 @@ public void testSecureOmInitFailures() throws Exception { FileUtils.deleteQuietly(Paths.get(securityConfig.getKeyLocation(COMPONENT) .toString(), securityConfig.getPublicKeyFileName()).toFile()); keyCodec.writePrivateKey(privateKey); - Assert.assertEquals(CertificateClient.InitResponse.SUCCESS, client.init()); - Assert.assertNotNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNotNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.SUCCESS, client.init()); + assertNotNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNotNull(client.getCertificate()); client.close(); // Case 7 When keypair and certificate is present. client = new OMCertificateClient( securityConfig, null, omStorage, omInfo, "", scmId, null, null); - Assert.assertEquals(CertificateClient.InitResponse.SUCCESS, client.init()); - Assert.assertNotNull(client.getPrivateKey()); - Assert.assertNotNull(client.getPublicKey()); - Assert.assertNotNull(client.getCertificate()); + assertEquals(CertificateClient.InitResponse.SUCCESS, client.init()); + assertNotNull(client.getPrivateKey()); + assertNotNull(client.getPublicKey()); + assertNotNull(client.getCertificate()); client.close(); } @@ -224,9 +222,9 @@ public void testSecureOmInitFailure() throws Exception { omStorage.setClusterId(clusterId); omStorage.setOmId(omId); config.set(OZONE_OM_ADDRESS_KEY, "om-unknown"); - RuntimeException rte = Assert.assertThrows(RuntimeException.class, + RuntimeException rte = assertThrows(RuntimeException.class, () -> OzoneManager.initializeSecurity(config, omStorage, scmId)); - Assert.assertEquals("Can't get SCM signed certificate. omRpcAdd:" + + assertEquals("Can't get SCM signed certificate. omRpcAdd:" + " om-unknown:9862", rte.getMessage()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index 714d7e5bec2..e311bc0b5e6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -38,10 +38,9 @@ import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils.VoidCallable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import java.io.IOException; import java.util.UUID; @@ -52,6 +51,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MULTITENANCY_ENABLED; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests that S3 requests for a tenant are directed to that tenant's volume, @@ -68,7 +69,7 @@ public class TestMultiTenantVolume { private static final String ACCESS_ID = "tenant$username"; private static OzoneClient client; - @BeforeClass + @BeforeAll public static void initClusterProvider() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setBoolean( @@ -85,7 +86,7 @@ public static void initClusterProvider() throws Exception { finalizeOMUpgrade(); } - @AfterClass + @AfterAll public static void shutdownClusterProvider() { IOUtils.closeQuietly(client); cluster.shutdown(); @@ -126,11 +127,11 @@ private static void preFinalizationChecks(ObjectStore store) // S3 get/set/revoke secret APIs still work before finalization final String accessId = "testUser1accessId1"; S3SecretValue s3SecretValue = store.getS3Secret(accessId); - Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey()); + assertEquals(accessId, s3SecretValue.getAwsAccessKey()); final String setSecret = "testsecret"; s3SecretValue = store.setS3Secret(accessId, setSecret); - Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey()); - Assert.assertEquals(setSecret, s3SecretValue.getAwsSecret()); + assertEquals(accessId, s3SecretValue.getAwsAccessKey()); + assertEquals(setSecret, s3SecretValue.getAwsSecret()); store.revokeS3Secret(accessId); } @@ -149,7 +150,7 @@ private static void finalizeOMUpgrade() omClient.finalizeUpgrade(upgradeClientID); // The status should transition as soon as the client call above returns - Assert.assertTrue(isStarting(finalizationResponse.status())); + assertTrue(isStarting(finalizationResponse.status())); // Wait for the finalization to be marked as done. // 10s timeout should be plenty. @@ -160,7 +161,7 @@ private static void finalizeOMUpgrade() upgradeClientID, false, false); return isDone(progress.status()); } catch (IOException e) { - Assert.fail("Unexpected exception while waiting for " + fail("Unexpected exception while waiting for " + "the OM upgrade to finalize: " + e.getMessage()); } return false; @@ -173,11 +174,11 @@ public void testDefaultS3Volume() throws Exception { // Default client not belonging to a tenant should end up in the S3 volume. ObjectStore store = client.getObjectStore(); - Assert.assertEquals(s3VolumeName, store.getS3Volume().getName()); + assertEquals(s3VolumeName, store.getS3Volume().getName()); // Create bucket. store.createS3Bucket(bucketName); - Assert.assertEquals(s3VolumeName, + assertEquals(s3VolumeName, store.getS3Bucket(bucketName).getVolumeName()); // Delete bucket. @@ -194,12 +195,12 @@ public void testS3TenantVolume() throws Exception { store.tenantAssignUserAccessId(USER_PRINCIPAL, TENANT_ID, ACCESS_ID); // S3 volume pointed to by the store should be for the tenant. - Assert.assertEquals(TENANT_ID, store.getS3Volume().getName()); + assertEquals(TENANT_ID, store.getS3Volume().getName()); // Create bucket in the tenant volume. store.createS3Bucket(BUCKET_NAME); OzoneBucket bucket = store.getS3Bucket(BUCKET_NAME); - Assert.assertEquals(TENANT_ID, bucket.getVolumeName()); + assertEquals(TENANT_ID, bucket.getVolumeName()); // A different user should not see bucket, since they will be directed to // the s3 volume. @@ -267,7 +268,7 @@ public void testOMRangerBGSyncRatisSetVersion() .get(OzoneConsts.RANGER_OZONE_SERVICE_VERSION_KEY); long readBackVersion = Long.parseLong(readBackVersionStr); - Assert.assertEquals(writtenVersion, readBackVersion); + assertEquals(writtenVersion, readBackVersion); } @Test @@ -279,8 +280,8 @@ public void testTenantVolumeQuota() throws Exception { store.createTenant(TENANT_ID); OzoneVolume volume; volume = store.getVolume(TENANT_ID); - Assert.assertEquals(OzoneConsts.QUOTA_RESET, volume.getQuotaInNamespace()); - Assert.assertEquals(OzoneConsts.QUOTA_RESET, volume.getQuotaInBytes()); + assertEquals(OzoneConsts.QUOTA_RESET, volume.getQuotaInNamespace()); + assertEquals(OzoneConsts.QUOTA_RESET, volume.getQuotaInBytes()); long spaceQuota = 10; long namespaceQuota = 20; @@ -289,8 +290,8 @@ public void testTenantVolumeQuota() throws Exception { // Check quota volume = store.getVolume(TENANT_ID); - Assert.assertEquals(namespaceQuota, volume.getQuotaInNamespace()); - Assert.assertEquals(spaceQuota, volume.getQuotaInBytes()); + assertEquals(namespaceQuota, volume.getQuotaInNamespace()); + assertEquals(spaceQuota, volume.getQuotaInBytes()); // Delete tenant and volume store.deleteTenant(TENANT_ID); @@ -309,8 +310,7 @@ public void testRejectNonS3CompliantTenantIdCreationWithDefaultStrictS3True() OMException.class, () -> store.createTenant(tenantId)); - Assert.assertTrue(e.getMessage().contains("Invalid volume name: " - + tenantId)); + assertTrue(e.getMessage().contains("Invalid volume name: " + tenantId)); } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java index 2208387324d..091471fab17 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java @@ -37,18 +37,18 @@ import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.apache.ozone.test.JUnit5AwareTimeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + + +import static org.junit.jupiter.api.Assertions.assertEquals; /** * Test OM's snapshot provider service. */ +@Timeout(300) public class TestOzoneManagerSnapshotProvider { private MiniOzoneHAClusterImpl cluster = null; @@ -59,14 +59,12 @@ public class TestOzoneManagerSnapshotProvider { private String omServiceId; private int numOfOMs = 3; - @Rule - public TestRule timeout = new JUnit5AwareTimeout(Timeout.seconds(300)); private OzoneClient client; /** * Create a MiniDFSCluster for testing. */ - @Before + @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); clusterId = UUID.randomUUID().toString(); @@ -88,7 +86,7 @@ public void init() throws Exception { /** * Shutdown MiniDFSCluster. */ - @After + @AfterEach public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { @@ -132,9 +130,9 @@ public void testDownloadCheckpoint() throws Exception { // The snapshot index downloaded from leader OM should match the ratis // snapshot index on the leader OM - Assert.assertEquals("The snapshot index downloaded from leader OM does " + - "not match its ratis snapshot index", - leaderSnapshotIndex, downloadedSnapshotIndex); + assertEquals(leaderSnapshotIndex, downloadedSnapshotIndex, + "The snapshot index downloaded from leader OM " + + "does not match its ratis snapshot index"); } private long getDownloadedSnapshotIndex(DBCheckpoint dbCheckpoint) From cc56c0ba52f98bcd23001d1d316611f455b91af1 Mon Sep 17 00:00:00 2001 From: Ivan Andika <36403683+ivandika3@users.noreply.github.com> Date: Tue, 5 Dec 2023 08:34:32 +0800 Subject: [PATCH 50/51] HDDS-9821. XceiverServerRatis SyncTimeoutRetry is overridden (#5717) --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 - .../apache/hadoop/ozone/OzoneConfigKeys.java | 3 - .../src/main/resources/ozone-default.xml | 11 +- .../server/ratis/XceiverServerRatis.java | 101 ++++++++++-------- 4 files changed, 68 insertions(+), 49 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 4b586b796d0..1eb15b28488 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -80,8 +80,6 @@ public final class ScmConfigKeys { public static final String DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES = "dfs.container.ratis.statemachinedata.sync.retries"; - public static final int - DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT = -1; public static final String DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS = "dfs.container.ratis.statemachine.max.pending.apply-transactions"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 89c9be54675..f124e24141f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -381,9 +381,6 @@ public final class OzoneConfigKeys { public static final String DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES = ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES; - public static final int - DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT = - ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT; public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS = ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS; public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS_DEFAULT = diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 7d8f5381789..bf9a2f511b6 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -98,10 +98,17 @@ dfs.container.ratis.statemachinedata.sync.retries - -1 + OZONE, DEBUG, CONTAINER, RATIS Number of times the WriteStateMachineData op will be tried - before failing, if this value is -1, then this retries indefinitely. + before failing. If the value is not configured, it will default + to (hdds.ratis.rpc.slowness.timeout / dfs.container.ratis.statemachinedata.sync.timeout), + which means that the WriteStatMachineData will be retried for every sync timeout until + the configured slowness timeout is hit, after which the StateMachine will close down the pipeline. + + If this value is set to -1, then this retries indefinitely. This might not be desirable + since if due to persistent failure the WriteStateMachineData op was not able to complete + for a long time, this might block the Ratis write pipeline. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index c9ac85414a7..4688ce4b278 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -265,41 +265,13 @@ public RaftProperties newRaftProperties() { final long raftSegmentPreallocatedSize = setRaftSegmentPreallocatedSize(properties); - TimeUnit timeUnit; - long duration; - - // set the configs enable and set the stateMachineData sync timeout - RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true); + // setup ratis stream if datastream is enabled if (streamEnable) { setUpRatisStream(properties); } - timeUnit = OzoneConfigKeys. - DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit(); - duration = conf.getTimeDuration( - OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT, - OzoneConfigKeys. - DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT - .getDuration(), timeUnit); - final TimeDuration dataSyncTimeout = - TimeDuration.valueOf(duration, timeUnit); - RaftServerConfigKeys.Log.StateMachineData - .setSyncTimeout(properties, dataSyncTimeout); - // typically a pipeline close will be initiated after a node failure - // timeout from Ratis in case a follower does not respond. - // By this time, all the writeStateMachine calls should be stopped - // and IOs should fail. - // Even if the leader is not able to complete write calls within - // the timeout seconds, it should just fail the operation and trigger - // pipeline close. failing the writeStateMachine call with limited retries - // will ensure even the leader initiates a pipeline close if its not - // able to complete write in the timeout configured. - - // NOTE : the default value for the retry count in ratis is -1, - // which means retry indefinitely. - RaftServerConfigKeys.Log.StateMachineData - .setSyncTimeoutRetry(properties, (int) nodeFailureTimeoutMs / - dataSyncTimeout.toIntExact(TimeUnit.MILLISECONDS)); + // Set Ratis State Machine Data configurations + setStateMachineDataConfigurations(properties); // set timeout for a retry cache entry setTimeoutForRetryCache(properties); @@ -359,17 +331,6 @@ public RaftProperties newRaftProperties() { RaftServerConfigKeys.Log.setQueueByteLimit(properties, SizeInBytes.valueOf(logQueueByteLimit)); - int numSyncRetries = conf.getInt( - OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES, - OzoneConfigKeys. - DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT); - RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties, - numSyncRetries); - - // Enable the StateMachineCaching - RaftServerConfigKeys.Log.StateMachineData.setCachingEnabled( - properties, true); - RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties, false); @@ -470,6 +431,62 @@ private void setRaftSegmentAndWriteBufferSize(RaftProperties properties) { SizeInBytes.valueOf(raftSegmentBufferSize)); } + private void setStateMachineDataConfigurations(RaftProperties properties) { + // set the configs enable and set the stateMachineData sync timeout + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true); + + TimeUnit timeUnit = OzoneConfigKeys. + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit(); + long duration = conf.getTimeDuration( + OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT, + OzoneConfigKeys. + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT + .getDuration(), timeUnit); + final TimeDuration dataSyncTimeout = + TimeDuration.valueOf(duration, timeUnit); + RaftServerConfigKeys.Log.StateMachineData + .setSyncTimeout(properties, dataSyncTimeout); + // typically a pipeline close will be initiated after a node failure + // timeout from Ratis in case a follower does not respond. + // By this time, all the writeStateMachine calls should be stopped + // and IOs should fail. + // Even if the leader is not able to complete write calls within + // the timeout seconds, it should just fail the operation and trigger + // pipeline close. failing the writeStateMachine call with limited retries + // will ensure even the leader initiates a pipeline close if its not + // able to complete write in the timeout configured. + + // NOTE : the default value for the retry count in ratis is -1, + // which means retry indefinitely. + int syncTimeoutRetryDefault = (int) nodeFailureTimeoutMs / + dataSyncTimeout.toIntExact(TimeUnit.MILLISECONDS); + int numSyncRetries = conf.getInt( + OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES, + syncTimeoutRetryDefault); + RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties, + numSyncRetries); + + // Enable the StateMachineCaching + // By enabling caching, the state machine data (e.g. write chunk data) + // will not be cached in Ratis log cache. The caching + // responsibility is deferred to the StateMachine implementation itself. + // ContainerStateMachine contains stateMachineDataCache that stores + // write chunk data for each log entry index. + // + // Note that in Ratis, the state machine data is never stored as + // part of the persisted Raft log entry. This means that the state + // machine data (in this case, the write chunk data) is only stored in the + // stateMachineDataCache until it's persisted in datanode storage + // (See ContainerStateMachine#writeStateMachineData) + // + // This requires ContainerStateMachine to implements additional mechanisms + // such as returning the state machine data in StateMachine#read to + // read back the state machine data that will be sent to the Ratis + // followers. + RaftServerConfigKeys.Log.StateMachineData.setCachingEnabled( + properties, true); + } + private RpcType setRpcType(RaftProperties properties) { final String rpcType = conf.get( OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, From 200b330c47908e21fe1b97efea1299ae198de306 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Tue, 5 Dec 2023 08:55:42 +0530 Subject: [PATCH 51/51] HDDS-9695. Do not show empty containers as missing in Recon UI (#5620) --- .../hadoop/ozone/recon/TestReconTasks.java | 125 +++++++- .../schema/ContainerSchemaDefinition.java | 1 + .../hadoop/ozone/recon/ReconConstants.java | 3 + .../recon/fsck/ContainerHealthStatus.java | 28 +- .../ozone/recon/fsck/ContainerHealthTask.java | 177 ++++++++-- .../ReconStorageContainerManagerFacade.java | 6 +- .../recon/fsck/TestContainerHealthStatus.java | 21 +- .../recon/fsck/TestContainerHealthTask.java | 54 +++- ...estContainerHealthTaskRecordGenerator.java | 301 ++++++++++++++++-- 9 files changed, 644 insertions(+), 72 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java index 5ecf0890ac8..965dd2f5255 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java @@ -28,11 +28,16 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.recon.scm.ReconContainerManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig; +import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils; import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers; @@ -40,6 +45,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.event.Level; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; @@ -71,6 +77,8 @@ public void init() throws Exception { cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1) .includeRecon(true).build(); cluster.waitForClusterToBeReady(); + GenericTestUtils.setLogLevel(SCMDatanodeHeartbeatDispatcher.LOG, + Level.DEBUG); } @AfterEach @@ -120,6 +128,9 @@ public void testMissingContainerDownNode() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) cluster.getReconServer().getReconStorageContainerManager(); + ReconContainerMetadataManager reconContainerMetadataManager = + cluster.getReconServer().getReconContainerMetadataManager(); + StorageContainerManager scm = cluster.getStorageContainerManager(); PipelineManager reconPipelineManager = reconScm.getPipelineManager(); PipelineManager scmPipelineManager = scm.getPipelineManager(); @@ -135,6 +146,13 @@ public void testMissingContainerDownNode() throws Exception { scmContainerManager .allocateContainer(RatisReplicationConfig.getInstance(ONE), "test"); long containerID = containerInfo.getContainerID(); + + try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { + reconContainerMetadataManager + .batchStoreContainerKeyCounts(rdbBatchOperation, containerID, 2L); + reconContainerMetadataManager.commitBatchOperation(rdbBatchOperation); + } + Pipeline pipeline = scmPipelineManager.getPipeline(containerInfo.getPipelineID()); XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); @@ -147,7 +165,7 @@ public void testMissingContainerDownNode() throws Exception { // Bring down the Datanode that had the container replica. cluster.shutdownHddsDatanode(pipeline.getFirstNode()); - LambdaTestUtils.await(120000, 10000, () -> { + LambdaTestUtils.await(120000, 6000, () -> { List allMissingContainers = reconContainerManager.getContainerSchemaManager() .getUnhealthyContainers( @@ -166,5 +184,110 @@ public void testMissingContainerDownNode() throws Exception { 0, 1000); return (allMissingContainers.isEmpty()); }); + IOUtils.closeQuietly(client); + } + + /** + * This test verifies the count of MISSING and EMPTY_MISSING containers. + * Following steps being followed in a single DN cluster. + * --- Allocate a container in SCM. + * --- Client writes the chunk and put block to only DN successfully. + * --- Shuts down the only DN. + * --- Since container to key mapping doesn't have any key mapped to + * container, missing container will be marked EMPTY_MISSING. + * --- Add a key mapping entry to key container mapping table for the + * container added. + * --- Now container will no longer be marked as EMPTY_MISSING and just + * as MISSING. + * --- Restart the only DN in cluster. + * --- Now container no longer will be marked as MISSING. + * + * @throws Exception + */ + @Test + public void testEmptyMissingContainerDownNode() throws Exception { + ReconStorageContainerManagerFacade reconScm = + (ReconStorageContainerManagerFacade) + cluster.getReconServer().getReconStorageContainerManager(); + ReconContainerMetadataManager reconContainerMetadataManager = + cluster.getReconServer().getReconContainerMetadataManager(); + StorageContainerManager scm = cluster.getStorageContainerManager(); + PipelineManager reconPipelineManager = reconScm.getPipelineManager(); + PipelineManager scmPipelineManager = scm.getPipelineManager(); + + // Make sure Recon's pipeline state is initialized. + LambdaTestUtils.await(60000, 1000, + () -> (reconPipelineManager.getPipelines().size() >= 1)); + + ContainerManager scmContainerManager = scm.getContainerManager(); + ReconContainerManager reconContainerManager = + (ReconContainerManager) reconScm.getContainerManager(); + ContainerInfo containerInfo = + scmContainerManager + .allocateContainer(RatisReplicationConfig.getInstance(ONE), "test"); + long containerID = containerInfo.getContainerID(); + + Pipeline pipeline = + scmPipelineManager.getPipeline(containerInfo.getPipelineID()); + XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); + runTestOzoneContainerViaDataNode(containerID, client); + + // Make sure Recon got the container report with new container. + assertEquals(scmContainerManager.getContainers(), + reconContainerManager.getContainers()); + + // Bring down the Datanode that had the container replica. + cluster.shutdownHddsDatanode(pipeline.getFirstNode()); + + LambdaTestUtils.await(25000, 1000, () -> { + List allEmptyMissingContainers = + reconContainerManager.getContainerSchemaManager() + .getUnhealthyContainers( + ContainerSchemaDefinition.UnHealthyContainerStates. + EMPTY_MISSING, + 0, 1000); + return (allEmptyMissingContainers.size() == 1); + }); + + // Now add a container to key mapping count as 3. This data is used to + // identify if container is empty in terms of keys mapped to container. + try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { + reconContainerMetadataManager + .batchStoreContainerKeyCounts(rdbBatchOperation, containerID, 3L); + reconContainerMetadataManager.commitBatchOperation(rdbBatchOperation); + } + + // Verify again and now container is not empty missing but just missing. + LambdaTestUtils.await(25000, 1000, () -> { + List allMissingContainers = + reconContainerManager.getContainerSchemaManager() + .getUnhealthyContainers( + ContainerSchemaDefinition.UnHealthyContainerStates.MISSING, + 0, 1000); + return (allMissingContainers.size() == 1); + }); + + LambdaTestUtils.await(25000, 1000, () -> { + List allEmptyMissingContainers = + reconContainerManager.getContainerSchemaManager() + .getUnhealthyContainers( + ContainerSchemaDefinition.UnHealthyContainerStates. + EMPTY_MISSING, + 0, 1000); + return (allEmptyMissingContainers.isEmpty()); + }); + + // Now restart the cluster and verify the container is no longer missing. + cluster.restartHddsDatanode(pipeline.getFirstNode(), true); + LambdaTestUtils.await(25000, 1000, () -> { + List allMissingContainers = + reconContainerManager.getContainerSchemaManager() + .getUnhealthyContainers( + ContainerSchemaDefinition.UnHealthyContainerStates.MISSING, + 0, 1000); + return (allMissingContainers.isEmpty()); + }); + + IOUtils.closeQuietly(client); } } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java index 1a045636bb7..43e2d728b76 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java @@ -47,6 +47,7 @@ public class ContainerSchemaDefinition implements ReconSchemaDefinition { */ public enum UnHealthyContainerStates { MISSING, + EMPTY_MISSING, UNDER_REPLICATED, OVER_REPLICATED, MIS_REPLICATED, diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 5e2e1a837f0..134092146e5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -58,6 +58,9 @@ private ReconConstants() { public static final String RECON_ENTITY_PATH = "path"; public static final String RECON_ENTITY_TYPE = "entityType"; public static final String RECON_ACCESS_METADATA_START_DATE = "startDate"; + public static final String CONTAINER_COUNT = "CONTAINER_COUNT"; + public static final String TOTAL_KEYS = "TOTAL_KEYS"; + public static final String TOTAL_USED_BYTES = "TOTAL_USED_BYTES"; // 1125899906842624L = 1PB public static final long MAX_FILE_SIZE_UPPER_BOUND = 1125899906842624L; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java index b004abd4321..7785e01a373 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; + +import java.io.IOException; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -39,11 +42,16 @@ public class ContainerHealthStatus { private int replicaDelta; private Set healthyReplicas; private ContainerPlacementStatus placementStatus; + private ReconContainerMetadataManager reconContainerMetadataManager; private int numReplicas; + private long numKeys; ContainerHealthStatus(ContainerInfo container, Set healthyReplicas, - PlacementPolicy placementPolicy) { + PlacementPolicy placementPolicy, + ReconContainerMetadataManager + reconContainerMetadataManager) { + this.reconContainerMetadataManager = reconContainerMetadataManager; this.container = container; int repFactor = container.getReplicationConfig().getRequiredNodes(); this.healthyReplicas = healthyReplicas @@ -54,6 +62,7 @@ public class ContainerHealthStatus { this.replicaDelta = repFactor - this.healthyReplicas.size(); this.placementStatus = getPlacementStatus(placementPolicy, repFactor); this.numReplicas = healthyReplicas.size(); + this.numKeys = getContainerKeyCount(container.getContainerID()); } public long getContainerID() { @@ -117,6 +126,10 @@ public boolean isMissing() { return numReplicas == 0; } + public boolean isEmpty() { + return numKeys == 0; + } + private ContainerPlacementStatus getPlacementStatus( PlacementPolicy policy, int repFactor) { List dns = healthyReplicas.stream() @@ -124,4 +137,17 @@ private ContainerPlacementStatus getPlacementStatus( .collect(Collectors.toList()); return policy.validateContainerPlacement(dns, repFactor); } + + private long getContainerKeyCount(long containerID) { + try { + return reconContainerMetadataManager.getKeyCountForContainer( + containerID); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public long getNumKeys() { + return numKeys; + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java index c5fc6ab625e..bb93923cfd1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java @@ -20,8 +20,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -37,6 +40,7 @@ import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.scm.ReconScmTask; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig; import org.apache.hadoop.util.Time; @@ -48,6 +52,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT; +import static org.apache.hadoop.ozone.recon.ReconConstants.TOTAL_KEYS; +import static org.apache.hadoop.ozone.recon.ReconConstants.TOTAL_USED_BYTES; + /** * Class that scans the list of containers and keeps track of containers with @@ -63,6 +71,7 @@ public class ContainerHealthTask extends ReconScmTask { private StorageContainerServiceProvider scmClient; private ContainerManager containerManager; private ContainerHealthSchemaManager containerHealthSchemaManager; + private ReconContainerMetadataManager reconContainerMetadataManager; private PlacementPolicy placementPolicy; private final long interval; @@ -74,10 +83,12 @@ public ContainerHealthTask( ReconTaskStatusDao reconTaskStatusDao, ContainerHealthSchemaManager containerHealthSchemaManager, PlacementPolicy placementPolicy, - ReconTaskConfig reconTaskConfig) { + ReconTaskConfig reconTaskConfig, + ReconContainerMetadataManager reconContainerMetadataManager) { super(reconTaskStatusDao); this.scmClient = scmClient; this.containerHealthSchemaManager = containerHealthSchemaManager; + this.reconContainerMetadataManager = reconContainerMetadataManager; this.placementPolicy = placementPolicy; this.containerManager = containerManager; interval = reconTaskConfig.getMissingContainerTaskInterval().toMillis(); @@ -100,10 +111,23 @@ public void run() { public void triggerContainerHealthCheck() { lock.writeLock().lock(); + // Map contains all UNHEALTHY STATES as keys and value is another map + // with 3 keys (CONTAINER_COUNT, TOTAL_KEYS, TOTAL_USED_BYTES) and value + // is count for each of these 3 stats. + // E.g. >, >, + // >, + // >, >, + // > + Map> + unhealthyContainerStateStatsMap; try { + unhealthyContainerStateStatsMap = new HashMap<>(Collections.emptyMap()); + initializeUnhealthyContainerStateStatsMap( + unhealthyContainerStateStatsMap); long start = Time.monotonicNow(); long currentTime = System.currentTimeMillis(); - long existingCount = processExistingDBRecords(currentTime); + long existingCount = processExistingDBRecords(currentTime, + unhealthyContainerStateStatsMap); LOG.info("Container Health task thread took {} milliseconds to" + " process {} existing database records.", Time.monotonicNow() - start, existingCount); @@ -111,31 +135,77 @@ public void triggerContainerHealthCheck() { final List containers = containerManager.getContainers(); containers.stream() .filter(c -> !processedContainers.contains(c)) - .forEach(c -> processContainer(c, currentTime)); + .forEach(c -> processContainer(c, currentTime, + unhealthyContainerStateStatsMap)); recordSingleRunCompletion(); LOG.info("Container Health task thread took {} milliseconds for" + " processing {} containers.", Time.monotonicNow() - start, containers.size()); + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); processedContainers.clear(); } finally { lock.writeLock().unlock(); } } + private void logUnhealthyContainerStats( + Map> + unhealthyContainerStateStatsMap) { + // If any EMPTY_MISSING containers, then it is possible that such + // containers got stuck in the closing state which never got + // any replicas created on the datanodes. In this case, we log it as + // EMPTY, and insert as EMPTY_MISSING in UNHEALTHY_CONTAINERS table. + unhealthyContainerStateStatsMap.entrySet().forEach(stateEntry -> { + UnHealthyContainerStates unhealthyContainerState = stateEntry.getKey(); + Map containerStateStatsMap = stateEntry.getValue(); + StringBuilder logMsgBuilder = + new StringBuilder(unhealthyContainerState.toString()); + logMsgBuilder.append(" **Container State Stats:** \n\t"); + containerStateStatsMap.entrySet().forEach(statsEntry -> { + logMsgBuilder.append(statsEntry.getKey()); + logMsgBuilder.append(" -> "); + logMsgBuilder.append(statsEntry.getValue()); + logMsgBuilder.append(" , "); + }); + LOG.info(logMsgBuilder.toString()); + }); + } + + private void initializeUnhealthyContainerStateStatsMap( + Map> + unhealthyContainerStateStatsMap) { + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.MISSING, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.EMPTY_MISSING, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.UNDER_REPLICATED, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.OVER_REPLICATED, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.MIS_REPLICATED, new HashMap<>()); + } + private ContainerHealthStatus setCurrentContainer(long recordId) throws ContainerNotFoundException { ContainerInfo container = containerManager.getContainer(ContainerID.valueOf(recordId)); Set replicas = containerManager.getContainerReplicas(container.containerID()); - return new ContainerHealthStatus(container, replicas, placementPolicy); + return new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); } - private void completeProcessingContainer(ContainerHealthStatus container, - Set existingRecords, long currentTime) { + private void completeProcessingContainer( + ContainerHealthStatus container, + Set existingRecords, + long currentTime, + Map> + unhealthyContainerStateCountMap) { containerHealthSchemaManager.insertUnhealthyContainerRecords( ContainerHealthRecords.generateUnhealthyRecords( - container, existingRecords, currentTime)); + container, existingRecords, currentTime, + unhealthyContainerStateCountMap)); processedContainers.add(container.getContainer()); } @@ -151,9 +221,13 @@ private void completeProcessingContainer(ContainerHealthStatus container, * additional records need to be added to the database. * * @param currentTime Timestamp to place on all records generated by this run + * @param unhealthyContainerStateCountMap * @return Count of records processed */ - private long processExistingDBRecords(long currentTime) { + private long processExistingDBRecords(long currentTime, + Map> + unhealthyContainerStateCountMap) { long recordCount = 0; try (Cursor cursor = containerHealthSchemaManager.getAllUnhealthyRecordsCursor()) { @@ -168,7 +242,8 @@ private long processExistingDBRecords(long currentTime) { } if (currentContainer.getContainerID() != rec.getContainerId()) { completeProcessingContainer( - currentContainer, existingRecords, currentTime); + currentContainer, existingRecords, currentTime, + unhealthyContainerStateCountMap); existingRecords.clear(); currentContainer = setCurrentContainer(rec.getContainerId()); } @@ -195,18 +270,22 @@ private long processExistingDBRecords(long currentTime) { // Remember to finish processing the last container if (currentContainer != null) { completeProcessingContainer( - currentContainer, existingRecords, currentTime); + currentContainer, existingRecords, currentTime, + unhealthyContainerStateCountMap); } } return recordCount; } - private void processContainer(ContainerInfo container, long currentTime) { + private void processContainer(ContainerInfo container, long currentTime, + Map> + unhealthyContainerStateStatsMap) { try { Set containerReplicas = containerManager.getContainerReplicas(container.containerID()); - ContainerHealthStatus h = new ContainerHealthStatus( - container, containerReplicas, placementPolicy); + ContainerHealthStatus h = new ContainerHealthStatus(container, + containerReplicas, placementPolicy, reconContainerMetadataManager); if (h.isHealthy() || h.isDeleted()) { return; } @@ -215,7 +294,8 @@ private void processContainer(ContainerInfo container, long currentTime) { return; } containerHealthSchemaManager.insertUnhealthyContainerRecords( - ContainerHealthRecords.generateUnhealthyRecords(h, currentTime)); + ContainerHealthRecords.generateUnhealthyRecords(h, currentTime, + unhealthyContainerStateStatsMap)); } catch (ContainerNotFoundException e) { LOG.error("Container not found while processing container in Container " + "Health task", e); @@ -298,8 +378,11 @@ public static boolean retainOrUpdateRecord( } public static List generateUnhealthyRecords( - ContainerHealthStatus container, long time) { - return generateUnhealthyRecords(container, new HashSet<>(), time); + ContainerHealthStatus container, long time, + Map> + unhealthyContainerStateStatsMap) { + return generateUnhealthyRecords(container, new HashSet<>(), time, + unhealthyContainerStateStatsMap); } /** @@ -312,7 +395,9 @@ public static List generateUnhealthyRecords( */ public static List generateUnhealthyRecords( ContainerHealthStatus container, Set recordForStateExists, - long time) { + long time, + Map> + unhealthyContainerStateStatsMap) { List records = new ArrayList<>(); if (container.isHealthy() || container.isDeleted()) { return records; @@ -320,9 +405,32 @@ public static List generateUnhealthyRecords( if (container.isMissing() && !recordForStateExists.contains( - UnHealthyContainerStates.MISSING.toString())) { - records.add( - recordForState(container, UnHealthyContainerStates.MISSING, time)); + UnHealthyContainerStates.MISSING.toString())) { + if (!container.isEmpty()) { + LOG.info("Non-empty container {} is missing. It has {} " + + "keys and {} bytes used according to SCM metadata. " + + "Please visit Recon's missing container page for a list of " + + "keys (and their metadata) mapped to this container.", + container.getContainerID(), container.getNumKeys(), + container.getContainer().getUsedBytes()); + records.add( + recordForState(container, UnHealthyContainerStates.MISSING, + time)); + populateContainerStats(container, UnHealthyContainerStates.MISSING, + unhealthyContainerStateStatsMap); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty container {} is missing. Kindly check the " + + "consolidated container stats per UNHEALTHY state logged as " + + "starting with **Container State Stats:**"); + } + records.add( + recordForState(container, UnHealthyContainerStates.EMPTY_MISSING, + time)); + populateContainerStats(container, + UnHealthyContainerStates.EMPTY_MISSING, + unhealthyContainerStateStatsMap); + } // A container cannot have any other records if it is missing so return return records; } @@ -332,6 +440,9 @@ public static List generateUnhealthyRecords( UnHealthyContainerStates.UNDER_REPLICATED.toString())) { records.add(recordForState( container, UnHealthyContainerStates.UNDER_REPLICATED, time)); + populateContainerStats(container, + UnHealthyContainerStates.UNDER_REPLICATED, + unhealthyContainerStateStatsMap); } if (container.isOverReplicated() @@ -339,6 +450,9 @@ public static List generateUnhealthyRecords( UnHealthyContainerStates.OVER_REPLICATED.toString())) { records.add(recordForState( container, UnHealthyContainerStates.OVER_REPLICATED, time)); + populateContainerStats(container, + UnHealthyContainerStates.OVER_REPLICATED, + unhealthyContainerStateStatsMap); } if (container.isMisReplicated() @@ -346,6 +460,9 @@ public static List generateUnhealthyRecords( UnHealthyContainerStates.MIS_REPLICATED.toString())) { records.add(recordForState( container, UnHealthyContainerStates.MIS_REPLICATED, time)); + populateContainerStats(container, + UnHealthyContainerStates.MIS_REPLICATED, + unhealthyContainerStateStatsMap); } return records; @@ -440,4 +557,24 @@ private static void updateReason( } } } + + private static void populateContainerStats( + ContainerHealthStatus container, + UnHealthyContainerStates unhealthyState, + Map> + unhealthyContainerStateStatsMap) { + if (unhealthyContainerStateStatsMap.containsKey(unhealthyState)) { + Map containerStatsMap = + unhealthyContainerStateStatsMap.get(unhealthyState); + containerStatsMap.compute(CONTAINER_COUNT, + (containerCount, value) -> (value == null) ? 1 : (value + 1)); + containerStatsMap.compute(TOTAL_KEYS, + (totalKeyCount, value) -> (value == null) ? container.getNumKeys() : + (value + container.getNumKeys())); + containerStatsMap.compute(TOTAL_USED_BYTES, + (totalUsedBytes, value) -> (value == null) ? + container.getContainer().getUsedBytes() : + (value + container.getContainer().getUsedBytes())); + } + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 1c72d990b30..464ec1a5ee8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -239,9 +239,9 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, reconTaskStatusDao, reconTaskConfig); ContainerHealthTask containerHealthTask = new ContainerHealthTask( - containerManager, scmServiceProvider, - reconTaskStatusDao, containerHealthSchemaManager, - containerPlacementPolicy, reconTaskConfig); + containerManager, scmServiceProvider, reconTaskStatusDao, + containerHealthSchemaManager, containerPlacementPolicy, reconTaskConfig, + reconContainerMetadataManager); this.containerSizeCountTask = new ContainerSizeCountTask( containerManager, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java index 572e4d7f51e..4e86b72709b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -46,11 +47,13 @@ public class TestContainerHealthStatus { private PlacementPolicy placementPolicy; private ContainerInfo container; + private ReconContainerMetadataManager reconContainerMetadataManager; @BeforeEach public void setup() { placementPolicy = mock(PlacementPolicy.class); container = mock(ContainerInfo.class); + reconContainerMetadataManager = mock(ReconContainerMetadataManager.class); when(container.getReplicationConfig()) .thenReturn(RatisReplicationConfig .getInstance(HddsProtos.ReplicationFactor.THREE)); @@ -68,7 +71,8 @@ public void testHealthyContainer() { ContainerReplicaProto.State.CLOSED, ContainerReplicaProto.State.CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertTrue(status.isHealthy()); assertFalse(status.isOverReplicated()); assertFalse(status.isUnderReplicated()); @@ -91,7 +95,8 @@ public void testHealthyContainerWithExtraUnhealthyReplica() { ContainerReplicaProto.State.CLOSED, ContainerReplicaProto.State.UNHEALTHY); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertTrue(status.isHealthy()); assertFalse(status.isOverReplicated()); assertFalse(status.isUnderReplicated()); @@ -105,7 +110,8 @@ public void testHealthyContainerWithExtraUnhealthyReplica() { public void testMissingContainer() { Set replicas = new HashSet<>(); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(status.isHealthy()); assertFalse(status.isOverReplicated()); assertFalse(status.isUnderReplicated()); @@ -120,7 +126,8 @@ public void testUnderReplicatedContainer() { Set replicas = generateReplicas(container, ContainerReplicaProto.State.CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(status.isHealthy()); assertFalse(status.isMissing()); assertFalse(status.isOverReplicated()); @@ -138,7 +145,8 @@ public void testOverReplicatedContainer() { ContainerReplicaProto.State.CLOSED, ContainerReplicaProto.State.CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(status.isHealthy()); assertFalse(status.isMissing()); assertFalse(status.isUnderReplicated()); @@ -158,7 +166,8 @@ public void testMisReplicated() { Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(1, 2, 5)); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(status.isHealthy()); assertFalse(status.isMissing()); assertFalse(status.isUnderReplicated()); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java index 77cd4baccee..847b5d98c7e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig; import org.apache.ozone.test.LambdaTestUtils; @@ -78,6 +79,8 @@ public void testRun() throws Exception { unHealthyContainersTableHandle); ReconStorageContainerManagerFacade scmMock = mock(ReconStorageContainerManagerFacade.class); + ReconContainerMetadataManager reconContainerMetadataManager = + mock(ReconContainerMetadataManager.class); MockPlacementPolicy placementMock = new MockPlacementPolicy(); ContainerManager containerManagerMock = mock(ContainerManager.class); StorageContainerServiceProvider scmClientMock = @@ -87,9 +90,10 @@ public void testRun() throws Exception { ContainerReplica healthyReplicaMock = mock(ContainerReplica.class); when(healthyReplicaMock.getState()).thenReturn(State.CLOSED); - // Create 6 containers. The first 5 will have various unhealthy states - // defined below. The container with ID=6 will be healthy. - List mockContainers = getMockContainers(6); + // Create 7 containers. The first 5 will have various unhealthy states + // defined below. The container with ID=6 will be healthy and + // container with ID=7 will be EMPTY_MISSING + List mockContainers = getMockContainers(7); when(scmMock.getScmServiceProvider()).thenReturn(scmClientMock); when(scmMock.getContainerManager()).thenReturn(containerManagerMock); when(containerManagerMock.getContainers()).thenReturn(mockContainers); @@ -128,6 +132,10 @@ public void testRun() throws Exception { .thenReturn(getMockReplicas(6L, State.CLOSED, State.CLOSED, State.CLOSED)); + // return 0 replicas for container ID 7 -> EMPTY_MISSING + when(containerManagerMock.getContainerReplicas(ContainerID.valueOf(7L))) + .thenReturn(Collections.emptySet()); + List all = unHealthyContainersTableHandle.findAll(); Assertions.assertTrue(all.isEmpty()); @@ -135,14 +143,16 @@ public void testRun() throws Exception { ReconTaskStatusDao reconTaskStatusDao = getDao(ReconTaskStatusDao.class); ReconTaskConfig reconTaskConfig = new ReconTaskConfig(); reconTaskConfig.setMissingContainerTaskInterval(Duration.ofSeconds(2)); + when(reconContainerMetadataManager.getKeyCountForContainer( + 7L)).thenReturn(5L); ContainerHealthTask containerHealthTask = new ContainerHealthTask(scmMock.getContainerManager(), scmMock.getScmServiceProvider(), reconTaskStatusDao, containerHealthSchemaManager, - placementMock, reconTaskConfig); + placementMock, reconTaskConfig, reconContainerMetadataManager); containerHealthTask.start(); LambdaTestUtils.await(6000, 1000, () -> - (unHealthyContainersTableHandle.count() == 5)); + (unHealthyContainersTableHandle.count() == 6)); UnhealthyContainers rec = unHealthyContainersTableHandle.fetchByContainerId(1L).get(0); assertEquals("UNDER_REPLICATED", rec.getContainerState()); @@ -162,6 +172,10 @@ public void testRun() throws Exception { unhealthyContainers.get(0).getActualReplicaCount().intValue()); rec = unHealthyContainersTableHandle.fetchByContainerId(3L).get(0); + assertEquals("EMPTY_MISSING", rec.getContainerState()); + assertEquals(3, rec.getReplicaDelta().intValue()); + + rec = unHealthyContainersTableHandle.fetchByContainerId(7L).get(0); assertEquals("MISSING", rec.getContainerState()); assertEquals(3, rec.getReplicaDelta().intValue()); @@ -205,7 +219,7 @@ public void testRun() throws Exception { placementMock.setMisRepWhenDnPresent(null); LambdaTestUtils.await(6000, 1000, () -> - (unHealthyContainersTableHandle.count() == 3)); + (unHealthyContainersTableHandle.count() == 4)); rec = unHealthyContainersTableHandle.fetchByContainerId(1L).get(0); assertEquals("UNDER_REPLICATED", rec.getContainerState()); assertEquals(1, rec.getReplicaDelta().intValue()); @@ -215,7 +229,10 @@ public void testRun() throws Exception { unHealthyContainersTableHandle.fetchByContainerId(2L).size()); rec = unHealthyContainersTableHandle.fetchByContainerId(3L).get(0); + assertEquals("EMPTY_MISSING", rec.getContainerState()); + assertEquals(3, rec.getReplicaDelta().intValue()); + rec = unHealthyContainersTableHandle.fetchByContainerId(7L).get(0); assertEquals("MISSING", rec.getContainerState()); assertEquals(3, rec.getReplicaDelta().intValue()); @@ -243,10 +260,12 @@ public void testDeletedContainer() throws Exception { ContainerManager containerManagerMock = mock(ContainerManager.class); StorageContainerServiceProvider scmClientMock = mock(StorageContainerServiceProvider.class); + ReconContainerMetadataManager reconContainerMetadataManager = + mock(ReconContainerMetadataManager.class); // Create 2 containers. The first is OPEN will no replicas, the second is // CLOSED with no replicas. - List mockContainers = getMockContainers(2); + List mockContainers = getMockContainers(3); when(scmMock.getScmServiceProvider()).thenReturn(scmClientMock); when(scmMock.getContainerManager()).thenReturn(containerManagerMock); when(containerManagerMock.getContainers()).thenReturn(mockContainers); @@ -255,7 +274,7 @@ public void testDeletedContainer() throws Exception { when(scmClientMock.getContainerWithPipeline(c.getContainerID())) .thenReturn(new ContainerWithPipeline(c, null)); } - // Container State OPEN with no replicas + // Empty Container with OPEN State and no replicas when(containerManagerMock.getContainer(ContainerID.valueOf(1L)).getState()) .thenReturn(HddsProtos.LifeCycleState.OPEN); when(containerManagerMock.getContainerReplicas(ContainerID.valueOf(1L))) @@ -272,6 +291,14 @@ public void testDeletedContainer() throws Exception { when(scmClientMock.getContainerWithPipeline(2)) .thenReturn(new ContainerWithPipeline(mockDeletedContainer, null)); + // Container with OPEN State and no replicas + when(containerManagerMock.getContainer(ContainerID.valueOf(3L)).getState()) + .thenReturn(HddsProtos.LifeCycleState.OPEN); + when(containerManagerMock.getContainerReplicas(ContainerID.valueOf(3L))) + .thenReturn(Collections.emptySet()); + when(scmClientMock.getContainerWithPipeline(3)) + .thenReturn(new ContainerWithPipeline(mockContainers.get(0), null)); + List all = unHealthyContainersTableHandle.findAll(); Assertions.assertTrue(all.isEmpty()); @@ -279,19 +306,26 @@ public void testDeletedContainer() throws Exception { ReconTaskStatusDao reconTaskStatusDao = getDao(ReconTaskStatusDao.class); ReconTaskConfig reconTaskConfig = new ReconTaskConfig(); reconTaskConfig.setMissingContainerTaskInterval(Duration.ofSeconds(2)); + when(reconContainerMetadataManager.getKeyCountForContainer( + 1L)).thenReturn(5L); ContainerHealthTask containerHealthTask = new ContainerHealthTask(scmMock.getContainerManager(), scmMock.getScmServiceProvider(), reconTaskStatusDao, containerHealthSchemaManager, - placementMock, reconTaskConfig); + placementMock, reconTaskConfig, reconContainerMetadataManager); containerHealthTask.start(); LambdaTestUtils.await(6000, 1000, () -> - (unHealthyContainersTableHandle.count() == 1)); + (unHealthyContainersTableHandle.count() == 2)); UnhealthyContainers rec = unHealthyContainersTableHandle.fetchByContainerId(1L).get(0); assertEquals("MISSING", rec.getContainerState()); assertEquals(3, rec.getReplicaDelta().intValue()); + rec = + unHealthyContainersTableHandle.fetchByContainerId(3L).get(0); + assertEquals("EMPTY_MISSING", rec.getContainerState()); + assertEquals(3, rec.getReplicaDelta().intValue()); + ReconTaskStatus taskStatus = reconTaskStatusDao.findById(containerHealthTask.getTaskName()); Assertions.assertTrue(taskStatus.getLastUpdatedTimestamp() > diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java index 4e86ca90567..99a9961cc27 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java @@ -25,19 +25,26 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates; import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers; import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,20 +57,33 @@ * records to store in the database. */ public class TestContainerHealthTaskRecordGenerator { - + private static final Logger LOG = + LoggerFactory.getLogger(TestContainerHealthTaskRecordGenerator.class); private PlacementPolicy placementPolicy; private ContainerInfo container; + private ContainerInfo emptyContainer; + private ReconContainerMetadataManager reconContainerMetadataManager; @BeforeEach - public void setup() { + public void setup() throws IOException { placementPolicy = mock(PlacementPolicy.class); container = mock(ContainerInfo.class); + emptyContainer = mock(ContainerInfo.class); + reconContainerMetadataManager = mock(ReconContainerMetadataManager.class); when(container.getReplicationConfig()) .thenReturn( RatisReplicationConfig .getInstance(HddsProtos.ReplicationFactor.THREE)); when(container.containerID()).thenReturn(ContainerID.valueOf(123456)); when(container.getContainerID()).thenReturn((long)123456); + when(reconContainerMetadataManager.getKeyCountForContainer( + (long) 123456)).thenReturn(5L); + when(emptyContainer.getReplicationConfig()) + .thenReturn( + RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.THREE)); + when(emptyContainer.containerID()).thenReturn(ContainerID.valueOf(345678)); + when(emptyContainer.getContainerID()).thenReturn((long) 345678); when(placementPolicy.validateContainerPlacement( Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(1, 1, 1)); @@ -73,7 +93,8 @@ public void setup() { public void testMissingRecordRetained() { Set replicas = new HashSet<>(); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); // Missing record should be retained assertTrue(ContainerHealthTask.ContainerHealthRecords .retainOrUpdateRecord(status, missingRecord() @@ -91,7 +112,8 @@ public void testMissingRecordRetained() { )); replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(ContainerHealthTask.ContainerHealthRecords .retainOrUpdateRecord(status, missingRecord() )); @@ -103,7 +125,8 @@ public void testUnderReplicatedRecordRetainedAndUpdated() { Set replicas = generateReplicas(container, CLOSED, CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); UnhealthyContainersRecord rec = underReplicatedRecord(); assertTrue(ContainerHealthTask.ContainerHealthRecords @@ -125,7 +148,8 @@ public void testUnderReplicatedRecordRetainedAndUpdated() { // Container is now replicated OK - should be removed. replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(ContainerHealthTask.ContainerHealthRecords .retainOrUpdateRecord(status, rec)); } @@ -136,7 +160,8 @@ public void testOverReplicatedRecordRetainedAndUpdated() { Set replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED, CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); UnhealthyContainersRecord rec = overReplicatedRecord(); assertTrue(ContainerHealthTask.ContainerHealthRecords @@ -158,7 +183,8 @@ public void testOverReplicatedRecordRetainedAndUpdated() { // Container is now replicated OK - should be removed. replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(ContainerHealthTask.ContainerHealthRecords .retainOrUpdateRecord(status, rec)); } @@ -172,7 +198,8 @@ public void testMisReplicatedRecordRetainedAndUpdated() { Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(2, 3, 5)); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); UnhealthyContainersRecord rec = misReplicatedRecord(); assertTrue(ContainerHealthTask.ContainerHealthRecords @@ -197,31 +224,58 @@ public void testMisReplicatedRecordRetainedAndUpdated() { when(placementPolicy.validateContainerPlacement( Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(3, 3, 5)); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); assertFalse(ContainerHealthTask.ContainerHealthRecords .retainOrUpdateRecord(status, rec)); } @Test + @SuppressWarnings("checkstyle:methodlength") public void testCorrectRecordsGenerated() { Set replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED); - + Map> + unhealthyContainerStateStatsMap = + new HashMap<>(); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); // HEALTHY container - no records generated. ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); List records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, (long)1234567); + .generateUnhealthyRecords(status, (long) 1234567, + unhealthyContainerStateStatsMap); assertEquals(0, records.size()); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); // Over-replicated - expect 1 over replicated record replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED, CLOSED, CLOSED); status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, (long)1234567); + .generateUnhealthyRecords(status, (long) 1234567, + unhealthyContainerStateStatsMap); assertEquals(1, records.size()); UnhealthyContainers rec = records.get(0); assertEquals(UnHealthyContainerStates.OVER_REPLICATED.toString(), @@ -229,6 +283,24 @@ public void testCorrectRecordsGenerated() { assertEquals(3, rec.getExpectedReplicaCount().intValue()); assertEquals(5, rec.getActualReplicaCount().intValue()); assertEquals(-2, rec.getReplicaDelta().intValue()); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); // Under and Mis Replicated - expect 2 records - mis and under replicated replicas = @@ -237,10 +309,30 @@ public void testCorrectRecordsGenerated() { Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(1, 2, 5)); status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, (long)1234567); + .generateUnhealthyRecords(status, (long) 1234567, + unhealthyContainerStateStatsMap); assertEquals(2, records.size()); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); rec = findRecordForState(records, UnHealthyContainerStates.MIS_REPLICATED); assertEquals(UnHealthyContainerStates.MIS_REPLICATED.toString(), @@ -265,51 +357,161 @@ public void testCorrectRecordsGenerated() { Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(1, 2, 5)); status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, (long)1234567); + .generateUnhealthyRecords(status, (long) 1234567, + unhealthyContainerStateStatsMap); assertEquals(1, records.size()); rec = records.get(0); assertEquals(UnHealthyContainerStates.MISSING.toString(), rec.getContainerState()); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); + + status = + new ContainerHealthStatus(emptyContainer, replicas, placementPolicy, + reconContainerMetadataManager); + records = ContainerHealthTask.ContainerHealthRecords + .generateUnhealthyRecords(status, (long) 345678, + unhealthyContainerStateStatsMap); + assertEquals(1, records.size()); + rec = records.get(0); + assertEquals(UnHealthyContainerStates.EMPTY_MISSING.toString(), + rec.getContainerState()); + assertEquals(3, rec.getExpectedReplicaCount().intValue()); assertEquals(0, rec.getActualReplicaCount().intValue()); assertEquals(3, rec.getReplicaDelta().intValue()); + + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + unhealthyContainerStateStatsMap.clear(); } @Test public void testRecordNotGeneratedIfAlreadyExists() { + Map> + unhealthyContainerStateStatsMap = + new HashMap<>(); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); Set existingRec = new HashSet<>(); - for (UnHealthyContainerStates s : UnHealthyContainerStates.values()) { - existingRec.add(s.toString()); - } // Over-replicated Set replicas = generateReplicas( container, CLOSED, CLOSED, CLOSED, CLOSED, CLOSED); ContainerHealthStatus status = - new ContainerHealthStatus(container, replicas, placementPolicy); + new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); List records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, existingRec, (long)1234567); - assertEquals(0, records.size()); + .generateUnhealthyRecords(status, existingRec, (long) 1234567, + unhealthyContainerStateStatsMap); + assertEquals(1, records.size()); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); // Missing replicas.clear(); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, existingRec, (long)1234567); - assertEquals(0, records.size()); + .generateUnhealthyRecords(status, existingRec, (long) 1234567, + unhealthyContainerStateStatsMap); + assertEquals(1, records.size()); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + initializeUnhealthyContainerStateStatsMap(unhealthyContainerStateStatsMap); // Under and Mis-Replicated replicas = generateReplicas(container, CLOSED, CLOSED); when(placementPolicy.validateContainerPlacement( Mockito.anyList(), Mockito.anyInt())) .thenReturn(new ContainerPlacementStatusDefault(1, 2, 5)); - status = new ContainerHealthStatus(container, replicas, placementPolicy); + status = new ContainerHealthStatus(container, replicas, placementPolicy, + reconContainerMetadataManager); records = ContainerHealthTask.ContainerHealthRecords - .generateUnhealthyRecords(status, existingRec, (long)1234567); - assertEquals(0, records.size()); + .generateUnhealthyRecords(status, existingRec, (long) 1234567, + unhealthyContainerStateStatsMap); + assertEquals(2, records.size()); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.EMPTY_MISSING) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(0, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.OVER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.UNDER_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + assertEquals(1, unhealthyContainerStateStatsMap.get( + UnHealthyContainerStates.MIS_REPLICATED) + .getOrDefault(CONTAINER_COUNT, 0L)); + + logUnhealthyContainerStats(unhealthyContainerStateStatsMap); + unhealthyContainerStateStatsMap.clear(); } private UnhealthyContainers findRecordForState( @@ -359,4 +561,41 @@ private Set generateReplicas(ContainerInfo cont, return replicas; } + private void initializeUnhealthyContainerStateStatsMap( + Map> + unhealthyContainerStateStatsMap) { + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.MISSING, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.EMPTY_MISSING, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.UNDER_REPLICATED, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.OVER_REPLICATED, new HashMap<>()); + unhealthyContainerStateStatsMap.put( + UnHealthyContainerStates.MIS_REPLICATED, new HashMap<>()); + } + + private void logUnhealthyContainerStats( + Map> + unhealthyContainerStateStatsMap) { + // If any EMPTY_MISSING containers, then it is possible that such + // containers got stuck in the closing state which never got + // any replicas created on the datanodes. In this case, we log it as + // EMPTY, and insert as EMPTY_MISSING in UNHEALTHY_CONTAINERS table. + unhealthyContainerStateStatsMap.entrySet().forEach(stateEntry -> { + UnHealthyContainerStates unhealthyContainerState = stateEntry.getKey(); + Map containerStateStatsMap = stateEntry.getValue(); + StringBuilder logMsgBuilder = + new StringBuilder(unhealthyContainerState.toString()); + logMsgBuilder.append(" Container State Stats: \n\t"); + containerStateStatsMap.entrySet().forEach(statsEntry -> { + logMsgBuilder.append(statsEntry.getKey()); + logMsgBuilder.append(" -> "); + logMsgBuilder.append(statsEntry.getValue()); + logMsgBuilder.append(" , "); + }); + LOG.info(logMsgBuilder.toString()); + }); + } }