Merge remote-tracking branch 'origin/master' into HDDS-12007

hadoop-hdds/client/pom.xml

@@ -27,12 +27,10 @@
   <description>Apache Ozone Distributed Data Store Client Library</description>
 
   <dependencies>
-
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
-
     <dependency>
       <groupId>io.opentracing</groupId>
       <artifactId>opentracing-api</artifactId>
@@ -41,12 +39,10 @@
       <groupId>io.opentracing</groupId>
       <artifactId>opentracing-util</artifactId>
     </dependency>
-
     <dependency>
       <groupId>jakarta.annotation</groupId>
       <artifactId>jakarta.annotation-api</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
@@ -87,7 +83,6 @@
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-thirdparty-misc</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

hadoop-hdds/common/pom.xml

@@ -226,10 +226,10 @@
 
     <!-- Test dependencies -->
     <dependency>
+      <!-- Needed for mocking RaftServerImpl -->
       <groupId>com.codahale.metrics</groupId>
       <artifactId>metrics-core</artifactId>
       <scope>test</scope>
-      <!-- Needed for mocking RaftServerImpl -->
     </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java

@@ -18,7 +18,6 @@
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.Callable;
 
 import com.google.common.base.Strings;
 import org.apache.hadoop.fs.Path;
@@ -28,13 +27,13 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import picocli.CommandLine;
 import picocli.CommandLine.ExitCode;
-import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
 
 /**
  * This is a generic parent class for all the ozone related cli tools.
  */
-public class GenericCli implements Callable<Void>, GenericParentCommand {
+@CommandLine.Command
+public abstract class GenericCli implements GenericParentCommand {
 
   public static final int EXECUTION_ERROR_EXIT_CODE = -1;
 
@@ -71,15 +70,6 @@ public GenericCli(CommandLine.IFactory factory) {
     ExtensibleParentCommand.addSubcommands(cmd);
   }
 
-  /**
-   * Handle the error when subcommand is required but not set.
-   */
-  public static void missingSubcommand(CommandSpec spec) {
-    System.err.println("Incomplete command");
-    spec.commandLine().usage(System.err);
-    System.exit(EXECUTION_ERROR_EXIT_CODE);
-  }
-
   public void run(String[] argv) {
     int exitCode = execute(argv);
 
@@ -103,11 +93,6 @@ protected void printError(Throwable error) {
     }
   }
 
-  @Override
-  public Void call() throws Exception {
-    throw new MissingSubcommandException(cmd);
-  }
-
   @Override
   public OzoneConfiguration getOzoneConf() {
     return config;
@@ -121,7 +106,7 @@ public UserGroupInformation getUser() throws IOException {
   }
 
   @VisibleForTesting
-  public picocli.CommandLine getCmd() {
+  public CommandLine getCmd() {
     return cmd;
   }
 

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1892,6 +1892,48 @@
       interfaces by setting it to 0.0.0.0.</description>
   </property>
 
+  <property>
+    <name>ozone.s3g.webadmin.http.enabled</name>
+    <value>true</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>This option can be used to disable the web server which serves additional content in Ozone S3 Gateway.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.https-address</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>Ozone S3Gateway content server's HTTPS address and port.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.https-bind-host</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTPS server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.webadmin.https-address.
+      This is useful for making the Ozone S3Gateway HTTPS server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.http-address</name>
+    <value>0.0.0.0:19878</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The address and port where Ozone S3Gateway serves
+      web content.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTP server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.webadmin.http-address.
+      This is useful for making the Ozone S3Gateway HTTP server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
   <property>
     <name>ozone.s3g.http.auth.kerberos.principal</name>
     <value>HTTP/_HOST@REALM</value>
@@ -2045,15 +2087,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.om.ratis.enable</name>
-    <value>true</value>
-    <tag>OZONE, OM, RATIS, MANAGEMENT</tag>
-    <description>Property to enable or disable Ratis server on OM.
-    Please note - this is a temporary property to disable OM Ratis server.
-    </description>
-  </property>
-
   <property>
     <name>ozone.om.ratis.port</name>
     <value>9872</value>
@@ -3323,7 +3356,7 @@
   </property>
   <property>
     <name>ozone.recon.om.snapshot.task.interval.delay</name>
-    <value>10m</value>
+    <value>5s</value>
     <tag>OZONE, RECON, OM</tag>
     <description>
       Interval in MINUTES by Recon to request OM DB Snapshot.
@@ -3339,7 +3372,7 @@
   </property>
   <property>
     <name>recon.om.delta.update.limit</name>
-    <value>2000</value>
+    <value>50000</value>
     <tag>OZONE, RECON</tag>
     <description>
       Recon each time get a limited delta updates from OM.
@@ -3360,7 +3393,7 @@
   </property>
   <property>
     <name>recon.om.delta.update.loop.limit</name>
-    <value>10</value>
+    <value>50</value>
     <tag>OZONE, RECON</tag>
     <description>
       The sync between Recon and OM consists of several small

hadoop-hdds/container-service/pom.xml

@@ -24,8 +24,8 @@
   <packaging>jar</packaging>
   <name>Apache Ozone HDDS Container Service</name>
   <description>Apache Ozone Distributed Data Store Container Service</description>
-  <dependencies>
 
+  <dependencies>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
@@ -94,7 +94,6 @@
       <groupId>io.opentracing</groupId>
       <artifactId>opentracing-util</artifactId>
     </dependency>
-
     <dependency>
       <groupId>jakarta.annotation</groupId>
       <artifactId>jakarta.annotation-api</artifactId>
@@ -103,7 +102,6 @@
       <groupId>jakarta.xml.bind</groupId>
       <artifactId>jakarta.xml.bind-api</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-compress</artifactId>
@@ -168,7 +166,6 @@
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-proto</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-server</artifactId>
@@ -185,7 +182,6 @@
       <groupId>org.glassfish.jaxb</groupId>
       <artifactId>jaxb-runtime</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.rocksdb</groupId>
       <artifactId>rocksdbjni</artifactId>
@@ -194,12 +190,10 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.yaml</groupId>
       <artifactId>snakeyaml</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-docs</artifactId>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -26,6 +26,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -97,7 +98,7 @@
     hidden = true, description = "Start the datanode for ozone",
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true)
-public class HddsDatanodeService extends GenericCli implements ServicePlugin {
+public class HddsDatanodeService extends GenericCli implements Callable<Void>, ServicePlugin {
 
   private static final Logger LOG = LoggerFactory.getLogger(
       HddsDatanodeService.class);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLayoutVersion.java

@@ -34,6 +34,7 @@
  */
 public enum ContainerLayoutVersion {
 
+  @Deprecated /* Use FILE_PER_BLOCK instead */
   FILE_PER_CHUNK(1, "One file per chunk") {
     @Override
     public File getChunkFile(File chunkDir, BlockID blockID, String chunkName) {
@@ -47,7 +48,7 @@ public File getChunkFile(File chunkDir, BlockID blockID, String chunkName) {
     }
   };
 
-  private static final ContainerLayoutVersion
+  public static final ContainerLayoutVersion
       DEFAULT_LAYOUT = ContainerLayoutVersion.FILE_PER_BLOCK;
 
   private static final List<ContainerLayoutVersion> CONTAINER_LAYOUT_VERSIONS =

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java

@@ -40,6 +40,7 @@
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
@@ -124,6 +125,7 @@
 import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
 import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST;
+import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.DEFAULT_LAYOUT;
 import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
 
 import org.apache.hadoop.util.Time;
@@ -191,6 +193,14 @@ public KeyValueHandler(ConfigurationSource config,
     byteBufferToByteString =
         ByteStringConversion
             .createByteBufferConversion(isUnsafeByteBufferConversionEnabled);
+
+    if (ContainerLayoutVersion.getConfiguredVersion(conf) ==
+        ContainerLayoutVersion.FILE_PER_CHUNK) {
+      LOG.warn("FILE_PER_CHUNK layout is not supported. Falling back to default : {}.",
+          DEFAULT_LAYOUT.name());
+      OzoneConfiguration.of(conf).set(ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY,
+          DEFAULT_LAYOUT.name());
+    }
   }
 
   @VisibleForTesting

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java

@@ -57,6 +57,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
 import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -292,6 +293,13 @@ public void testVolumeSetInKeyValueHandler() throws Exception {
           keyValueHandler.getVolumeChoosingPolicyForTesting()
               .getClass().getName());
 
+      // Ensures that KeyValueHandler falls back to FILE_PER_BLOCK.
+      conf.set(OZONE_SCM_CONTAINER_LAYOUT_KEY, "FILE_PER_CHUNK");
+      new KeyValueHandler(conf, context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet,
+          metrics, c -> { });
+      assertEquals(ContainerLayoutVersion.FILE_PER_BLOCK,
+          conf.getEnum(OZONE_SCM_CONTAINER_LAYOUT_KEY, ContainerLayoutVersion.FILE_PER_CHUNK));
+
       //Set a class which is not of sub class of VolumeChoosingPolicy
       conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
           "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");