From 44516ff740139c0577b621021dcf47158876c9e0 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 7 Jan 2025 15:29:01 +0530 Subject: [PATCH 1/3] HDDS-12029. Move ozone debug recover to ozone admin om lease recover --- .../ozone/{debug => }/TestLeaseRecoverer.java | 6 ++-- .../apache/hadoop/ozone/admin/om/OMAdmin.java | 4 ++- .../om/lease}/LeaseRecoverer.java | 13 +++---- .../ozone/admin/om/lease/LeaseSubCommand.java | 35 +++++++++++++++++++ .../ozone/admin/om/lease/package-info.java | 22 ++++++++++++ 5 files changed, 67 insertions(+), 13 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{debug => }/TestLeaseRecoverer.java (96%) rename hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{debug => admin/om/lease}/LeaseRecoverer.java (83%) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseSubCommand.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/package-info.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java similarity index 96% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java index 29f91821ebd..7d2e8e2704a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java @@ -14,7 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package org.apache.hadoop.ozone.debug; +package org.apache.hadoop.ozone; import java.io.IOException; import java.io.PrintWriter; @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.ozone.admin.om.lease.LeaseRecoverer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -37,9 +38,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java index 2e6eedb9420..4d10132a7cb 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.admin.om.lease.LeaseSubCommand; import org.apache.hadoop.ozone.client.OzoneClientException; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; @@ -58,7 +59,8 @@ DecommissionOMSubcommand.class, UpdateRangerSubcommand.class, TransferOmLeaderSubCommand.class, - FetchKeySubCommand.class + FetchKeySubCommand.class, + LeaseSubCommand.class }) @MetaInfServices(AdminSubcommand.class) public class OMAdmin implements AdminSubcommand { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/LeaseRecoverer.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseRecoverer.java similarity index 83% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/LeaseRecoverer.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseRecoverer.java index 9c3865ae241..c62932747fb 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/LeaseRecoverer.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseRecoverer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.debug; +package org.apache.hadoop.ozone.admin.om.lease; import java.net.URI; import java.util.concurrent.Callable; @@ -24,24 +24,21 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.LeaseRecoverable; -import org.apache.hadoop.hdds.cli.DebugSubcommand; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.kohsuke.MetaInfServices; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; /** - * Tool that recover the lease of a specified file. + * CLI to recover the lease of a specified file. */ @CommandLine.Command( name = "recover", - customSynopsis = "ozone debug recover --path=", - description = "recover the lease of a specified file. Make sure to specify " + customSynopsis = "ozone admin om lease recover --path=", + description = "Recover the lease of a specified file. Make sure to specify " + "file system scheme if ofs:// is not the default.") -@MetaInfServices(DebugSubcommand.class) -public class LeaseRecoverer implements Callable, DebugSubcommand { +public class LeaseRecoverer implements Callable { @Spec private CommandSpec spec; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseSubCommand.java new file mode 100644 index 00000000000..001dccaf835 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/LeaseSubCommand.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.om.lease; + +import picocli.CommandLine; + +/** + * Handler of ozone admin om lease command. + */ +@CommandLine.Command( + name = "lease", + description = "Command for all lease related queries.", + subcommands = { + LeaseRecoverer.class + } +) +public class LeaseSubCommand { + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/package-info.java new file mode 100644 index 00000000000..a5cb707dd9a --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/lease/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Lease related OM Admin tools. + */ +package org.apache.hadoop.ozone.admin.om.lease; From 317e16a0783c8333603a55d9fbe1ffce50c3f5f7 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 9 Jan 2025 15:34:51 +0530 Subject: [PATCH 2/3] Corrected robot tests --- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 -- .../lease-recovery.robot} | 10 +++++++--- .../dist/src/main/smoketest/compatibility/read.robot | 2 +- .../dist/src/main/smoketest/debug/ozone-debug.robot | 5 ----- 4 files changed, 8 insertions(+), 11 deletions(-) rename hadoop-ozone/dist/src/main/smoketest/{debug/ozone-debug-lease-recovery.robot => admincli/lease-recovery.robot} (89%) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 6477dbd0979..a580fd83309 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -48,8 +48,6 @@ execute_robot_test scm freon execute_robot_test scm cli execute_robot_test scm admincli -execute_robot_test scm debug/ozone-debug-lease-recovery.robot - execute_robot_test scm -v USERNAME:httpfs httpfs execute_debug_tests diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/lease-recovery.robot similarity index 89% rename from hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot rename to hadoop-ozone/dist/src/main/smoketest/admincli/lease-recovery.robot index 691769dbd72..a44ceacc112 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/lease-recovery.robot @@ -18,7 +18,6 @@ Documentation Test lease recovery of ozone filesystem Library OperatingSystem Resource ../lib/os.robot Resource ../lib/fs.robot -Resource ozone-debug.robot Test Timeout 5 minute Suite Setup Create volume bucket and put key @@ -35,8 +34,13 @@ Create volume bucket and put key Create File ${TEMP_DIR}/${TESTFILE} Execute ozone sh key put /${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE} +Execute Lease recovery cli + [Arguments] ${KEY_PATH} + ${result} = Execute And Ignore Error ozone admin om lease recover --path=${KEY_PATH} + [Return] ${result} + *** Test Cases *** -Test ozone debug recover for o3fs +Test ozone admin om lease recover for o3fs ${o3fs_path} = Format FS URL o3fs ${VOLUME} ${BUCKET} ${TESTFILE} ${result} = Execute Lease recovery cli ${o3fs_path} Should Contain ${result} Lease recovery SUCCEEDED @@ -44,7 +48,7 @@ Test ozone debug recover for o3fs ${result} = Execute Lease recovery cli ${o3fs_path} Should Contain ${result} not found -Test ozone debug recover for ofs +Test ozone admin om lease recover for ofs ${ofs_path} = Format FS URL ofs ${VOLUME} ${BUCKET} ${TESTFILE} ${result} = Execute Lease recovery cli ${ofs_path} Should Contain ${result} Lease recovery SUCCEEDED diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot index 9061677eae6..b5dfbb9739e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot @@ -154,4 +154,4 @@ HSync Lease Recover Can Be Used Pass Execution If '${DATA_VERSION}' < '${FSO_VERSION}' Skipped write test case Pass Execution If '${CLIENT_VERSION}' < '${HSYNC_VERSION}' Client does not support HSYNC Pass Execution If '${CLUSTER_VERSION}' < '${HSYNC_VERSION}' Cluster does not support HSYNC - Execute ozone debug recover --path=ofs://om/vol1/fso-bucket-${DATA_VERSION}/dir/subdir/file + Execute ozone admin om lease recover --path=ofs://om/vol1/fso-bucket-${DATA_VERSION}/dir/subdir/file diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index fa915819eea..e0964e4c160 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -26,11 +26,6 @@ Execute read-replicas CLI tool File Should Exist ${directory}/${TESTFILE}_manifest [Return] ${directory} -Execute Lease recovery cli - [Arguments] ${KEY_PATH} - ${result} = Execute And Ignore Error ozone debug recover --path=${KEY_PATH} - [Return] ${result} - Read Replicas Manifest ${manifest} = Get File ${DIR}/${TESTFILE}_manifest ${json} = Evaluate json.loads('''${manifest}''') json From 6adb199f0c0e5d9598d0e064b3db4821c2543ee7 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 9 Jan 2025 19:51:23 +0530 Subject: [PATCH 3/3] Moved TestLeaseRecoverer to admin.om.lease package --- .../ozone/{ => admin/om/lease}/TestLeaseRecoverer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{ => admin/om/lease}/TestLeaseRecoverer.java (96%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java similarity index 96% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java index 7d2e8e2704a..004338483bb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestLeaseRecoverer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java @@ -14,7 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package org.apache.hadoop.ozone; +package org.apache.hadoop.ozone.admin.om.lease; import java.io.IOException; import java.io.PrintWriter; @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.hdds.scm.OzoneClientConfig; -import org.apache.hadoop.ozone.admin.om.lease.LeaseRecoverer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -38,6 +37,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout;