HDDS-11994. Convert Freon to pluggable model (#7620)

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/ExtensibleParentCommand.java

@@ -20,6 +20,8 @@
 import picocli.CommandLine;
 
 import java.util.ServiceLoader;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 /**
  * Interface for parent commands that accept subcommands to be dynamically registered.
@@ -40,11 +42,13 @@ static void addSubcommands(CommandLine cli) {
     if (command instanceof ExtensibleParentCommand) {
       ExtensibleParentCommand parentCommand = (ExtensibleParentCommand) command;
       ServiceLoader<?> subcommands = ServiceLoader.load(parentCommand.subcommandType());
+      SortedMap<String, CommandLine> sorted = new TreeMap<>();
       for (Object subcommand : subcommands) {
         final CommandLine.Command commandAnnotation = subcommand.getClass().getAnnotation(CommandLine.Command.class);
         CommandLine subcommandCommandLine = new CommandLine(subcommand, cli.getFactory());
-        cli.addSubcommand(commandAnnotation.name(), subcommandCommandLine);
+        sorted.put(commandAnnotation.name(), subcommandCommandLine);
       }
+      sorted.forEach(cli::addSubcommand);
     }
 
     // process subcommands recursively

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java

@@ -73,8 +73,9 @@
 /**
  * Base class for simplified performance tests.
  */
+@CommandLine.Command
 @SuppressWarnings("java:S2245") // no need for secure random
-public class BaseFreonGenerator {
+public class BaseFreonGenerator implements FreonSubcommand {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(BaseFreonGenerator.class);

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ChunkManagerDiskWrite.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
 
 import com.codahale.metrics.Timer;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -60,6 +61,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class ChunkManagerDiskWrite extends BaseFreonGenerator implements
     Callable<Void> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java

@@ -45,6 +45,7 @@
 import org.apache.hadoop.ozone.container.replication.ReplicationTask;
 import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
 import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
@@ -71,6 +72,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class ClosedContainerReplicator extends BaseFreonGenerator implements
     Callable<Void> {
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DNRPCLoadGenerator.java

@@ -37,6 +37,7 @@
 import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
 import org.apache.hadoop.ozone.util.PayloadUtils;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
@@ -59,6 +60,7 @@
         versionProvider = HddsVersionProvider.class,
         mixinStandardHelpOptions = true,
         showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class DNRPCLoadGenerator extends BaseFreonGenerator
         implements Callable<Void> {
   private static final Logger LOG =
@@ -111,7 +113,7 @@ public class DNRPCLoadGenerator extends BaseFreonGenerator
   private Freon freon;
 
   // empy constructor for picocli
-  DNRPCLoadGenerator() {
+  public DNRPCLoadGenerator() {
   }
 
   @VisibleForTesting

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeBlockPutter.java

@@ -39,6 +39,7 @@
 
 import com.codahale.metrics.Timer;
 import org.apache.commons.lang3.RandomStringUtils;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -55,6 +56,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class DatanodeBlockPutter extends BaseFreonGenerator implements
     Callable<Void> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkGenerator.java

@@ -48,6 +48,7 @@
 import com.codahale.metrics.Timer;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -62,6 +63,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class DatanodeChunkGenerator extends BaseFreonGenerator implements
     Callable<Void> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java

@@ -35,6 +35,7 @@
 
 import com.codahale.metrics.Timer;
 import org.apache.hadoop.ozone.common.OzoneChecksumException;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -49,6 +50,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class DatanodeChunkValidator extends BaseFreonGenerator
     implements Callable<Void> {
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java

@@ -57,6 +57,7 @@
 import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
@@ -119,7 +120,8 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
-public class DatanodeSimulator implements Callable<Void> {
+@MetaInfServices(FreonSubcommand.class)
+public class DatanodeSimulator implements Callable<Void>, FreonSubcommand {
   private static final Logger LOGGER = LoggerFactory.getLogger(
       DatanodeSimulator.class);
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FollowerAppendLogEntryGenerator.java

@@ -68,6 +68,7 @@
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
 import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
 import org.apache.ratis.util.Preconditions;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -88,6 +89,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class FollowerAppendLogEntryGenerator extends BaseAppendLogGenerator
     implements Callable<Void>, StreamObserver<AppendEntriesReplyProto> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java

@@ -18,14 +18,12 @@
 
 import java.io.IOException;
 
+import org.apache.hadoop.hdds.cli.ExtensibleParentCommand;
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.ozone.freon.containergenerator.GeneratorDatanode;
-import org.apache.hadoop.ozone.freon.containergenerator.GeneratorOm;
-import org.apache.hadoop.ozone.freon.containergenerator.GeneratorScm;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,47 +38,9 @@
 @Command(
     name = "ozone freon",
     description = "Load generator and tester tool for ozone",
-    subcommands = {
-        RandomKeyGenerator.class,
-        OzoneClientKeyGenerator.class,
-        OzoneClientKeyValidator.class,
-        OzoneClientKeyRemover.class,
-        OmKeyGenerator.class,
-        OmBucketGenerator.class,
-        OmBucketRemover.class,
-        HadoopFsGenerator.class,
-        HadoopNestedDirGenerator.class,
-        HadoopDirTreeGenerator.class,
-        HadoopFsValidator.class,
-        SameKeyReader.class,
-        S3KeyGenerator.class,
-        S3BucketGenerator.class,
-        DatanodeChunkGenerator.class,
-        DatanodeChunkValidator.class,
-        DatanodeBlockPutter.class,
-        FollowerAppendLogEntryGenerator.class,
-        ChunkManagerDiskWrite.class,
-        LeaderAppendLogEntryGenerator.class,
-        GeneratorOm.class,
-        GeneratorScm.class,
-        GeneratorDatanode.class,
-        ClosedContainerReplicator.class,
-        StreamingGenerator.class,
-        SCMThroughputBenchmark.class,
-        OmBucketReadWriteFileOps.class,
-        OmBucketReadWriteKeyOps.class,
-        OmRPCLoadGenerator.class,
-        OzoneClientKeyReadWriteListOps.class,
-        RangeKeysGenerator.class,
-        DatanodeSimulator.class,
-        OmMetadataGenerator.class,
-        DNRPCLoadGenerator.class,
-        HsyncGenerator.class,
-        OzoneClientCreator.class,
-    },
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true)
-public class Freon extends GenericCli {
+public class Freon extends GenericCli implements ExtensibleParentCommand {
 
   public static final Logger LOG = LoggerFactory.getLogger(Freon.class);
 
@@ -102,6 +62,11 @@ public int execute(String[] argv) {
     return super.execute(argv);
   }
 
+  @Override
+  public Class<?> subcommandType() {
+    return FreonSubcommand.class;
+  }
+
   public void stopHttpServer() {
     if (freonHttpServer != null) {
       try {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonSubcommand.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+/** Marker interface for subcommands to be registered for {@code ozone freon}. */
+public interface FreonSubcommand {
+  // marker
+}

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.StorageSize;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -41,6 +42,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class HadoopDirTreeGenerator extends HadoopBaseFreonGenerator
     implements Callable<Void> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java

@@ -25,6 +25,7 @@
 
 import com.codahale.metrics.Timer;
 import org.apache.hadoop.hdds.conf.StorageSize;
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
@@ -37,6 +38,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class HadoopFsGenerator extends HadoopBaseFreonGenerator
     implements Callable<Void> {
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java

@@ -25,6 +25,7 @@
 
 import com.codahale.metrics.Timer;
 import org.apache.commons.io.IOUtils;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -38,6 +39,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class HadoopFsValidator extends HadoopBaseFreonGenerator
     implements Callable<Void> {
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 
 import org.apache.commons.lang3.RandomStringUtils;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -37,6 +38,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class HadoopNestedDirGenerator extends HadoopBaseFreonGenerator
     implements Callable<Void> {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.util.PayloadUtils;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
@@ -57,6 +58,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class HsyncGenerator extends BaseFreonGenerator implements Callable<Void> {
   private static final Logger LOG = LoggerFactory.getLogger(HsyncGenerator.class);
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java

@@ -63,6 +63,7 @@
 import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
 import org.apache.ratis.thirdparty.io.grpc.netty.NegotiationType;
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
+import org.kohsuke.MetaInfServices;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -81,6 +82,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 @SuppressWarnings("java:S2245") // no need for secure random
 public class LeaderAppendLogEntryGenerator extends BaseAppendLogGenerator
     implements

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
 
 import com.codahale.metrics.Timer;
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
@@ -38,6 +39,7 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
+@MetaInfServices(FreonSubcommand.class)
 public class OmBucketGenerator extends BaseFreonGenerator
     implements Callable<Void> {