Merge branch 'master' into omDelMetrics

Tejaskriya · Jan 8, 2025 · 9c72d54 · 9c72d54
2 parents ecb123c + f8394cf
commit 9c72d54
Show file tree

Hide file tree

Showing 172 changed files with 1,421 additions and 1,481 deletions.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
@@ -18,7 +18,6 @@
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.Callable;
 
 import com.google.common.base.Strings;
 import org.apache.hadoop.fs.Path;
@@ -28,13 +27,13 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import picocli.CommandLine;
 import picocli.CommandLine.ExitCode;
-import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
 
 /**
  * This is a generic parent class for all the ozone related cli tools.
  */
-public class GenericCli implements Callable<Void>, GenericParentCommand {
+@CommandLine.Command
+public abstract class GenericCli implements GenericParentCommand {
 
   public static final int EXECUTION_ERROR_EXIT_CODE = -1;
 
@@ -71,15 +70,6 @@ public GenericCli(CommandLine.IFactory factory) {
     ExtensibleParentCommand.addSubcommands(cmd);
   }
 
-  /**
-   * Handle the error when subcommand is required but not set.
-   */
-  public static void missingSubcommand(CommandSpec spec) {
-    System.err.println("Incomplete command");
-    spec.commandLine().usage(System.err);
-    System.exit(EXECUTION_ERROR_EXIT_CODE);
-  }
-
   public void run(String[] argv) {
     int exitCode = execute(argv);
 
@@ -103,11 +93,6 @@ protected void printError(Throwable error) {
     }
   }
 
-  @Override
-  public Void call() throws Exception {
-    throw new MissingSubcommandException(cmd);
-  }
-
   @Override
   public OzoneConfiguration getOzoneConf() {
     return config;
@@ -121,7 +106,7 @@ public UserGroupInformation getUser() throws IOException {
   }
 
   @VisibleForTesting
-  public picocli.CommandLine getCmd() {
+  public CommandLine getCmd() {
     return cmd;
   }
 

diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1892,6 +1892,48 @@
       interfaces by setting it to 0.0.0.0.</description>
   </property>
 
+  <property>
+    <name>ozone.s3g.webadmin.http.enabled</name>
+    <value>true</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>This option can be used to disable the web server which serves additional content in Ozone S3 Gateway.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.https-address</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>Ozone S3Gateway content server's HTTPS address and port.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.https-bind-host</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTPS server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.webadmin.https-address.
+      This is useful for making the Ozone S3Gateway HTTPS server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.http-address</name>
+    <value>0.0.0.0:19878</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The address and port where Ozone S3Gateway serves
+      web content.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.webadmin.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTP server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.webadmin.http-address.
+      This is useful for making the Ozone S3Gateway HTTP server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
   <property>
     <name>ozone.s3g.http.auth.kerberos.principal</name>
     <value>HTTP/_HOST@REALM</value>
@@ -2045,15 +2087,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.om.ratis.enable</name>
-    <value>true</value>
-    <tag>OZONE, OM, RATIS, MANAGEMENT</tag>
-    <description>Property to enable or disable Ratis server on OM.
-    Please note - this is a temporary property to disable OM Ratis server.
-    </description>
-  </property>
-
   <property>
     <name>ozone.om.ratis.port</name>
     <value>9872</value>
@@ -3323,7 +3356,7 @@
   </property>
   <property>
     <name>ozone.recon.om.snapshot.task.interval.delay</name>
-    <value>10m</value>
+    <value>5s</value>
     <tag>OZONE, RECON, OM</tag>
     <description>
       Interval in MINUTES by Recon to request OM DB Snapshot.
@@ -3339,7 +3372,7 @@
   </property>
   <property>
     <name>recon.om.delta.update.limit</name>
-    <value>2000</value>
+    <value>50000</value>
     <tag>OZONE, RECON</tag>
     <description>
       Recon each time get a limited delta updates from OM.
@@ -3360,7 +3393,7 @@
   </property>
   <property>
     <name>recon.om.delta.update.loop.limit</name>
-    <value>10</value>
+    <value>50</value>
     <tag>OZONE, RECON</tag>
     <description>
       The sync between Recon and OM consists of several small

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -26,6 +26,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -97,7 +98,7 @@
     hidden = true, description = "Start the datanode for ozone",
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true)
-public class HddsDatanodeService extends GenericCli implements ServicePlugin {
+public class HddsDatanodeService extends GenericCli implements Callable<Void>, ServicePlugin {
 
   private static final Logger LOG = LoggerFactory.getLogger(
       HddsDatanodeService.class);

diff --git a/...e/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLayoutVersion.java b/...e/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLayoutVersion.java
@@ -34,6 +34,7 @@
  */
 public enum ContainerLayoutVersion {
 
+  @Deprecated /* Use FILE_PER_BLOCK instead */
   FILE_PER_CHUNK(1, "One file per chunk") {
     @Override
     public File getChunkFile(File chunkDir, BlockID blockID, String chunkName) {
@@ -47,7 +48,7 @@ public File getChunkFile(File chunkDir, BlockID blockID, String chunkName) {
     }
   };
 
-  private static final ContainerLayoutVersion
+  public static final ContainerLayoutVersion
       DEFAULT_LAYOUT = ContainerLayoutVersion.FILE_PER_BLOCK;
 
   private static final List<ContainerLayoutVersion> CONTAINER_LAYOUT_VERSIONS =

diff --git a/...ner-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/...ner-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
@@ -124,6 +125,7 @@
 import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
 import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST;
+import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.DEFAULT_LAYOUT;
 import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
 
 import org.apache.hadoop.util.Time;
@@ -191,6 +193,14 @@ public KeyValueHandler(ConfigurationSource config,
     byteBufferToByteString =
         ByteStringConversion
             .createByteBufferConversion(isUnsafeByteBufferConversionEnabled);
+
+    if (ContainerLayoutVersion.getConfiguredVersion(conf) ==
+        ContainerLayoutVersion.FILE_PER_CHUNK) {
+      LOG.warn("FILE_PER_CHUNK layout is not supported. Falling back to default : {}.",
+          DEFAULT_LAYOUT.name());
+      OzoneConfiguration.of(conf).set(ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY,
+          DEFAULT_LAYOUT.name());
+    }
   }
 
   @VisibleForTesting

diff --git a/...service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/...service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -57,6 +57,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
 import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -292,6 +293,13 @@ public void testVolumeSetInKeyValueHandler() throws Exception {
           keyValueHandler.getVolumeChoosingPolicyForTesting()
               .getClass().getName());
 
+      // Ensures that KeyValueHandler falls back to FILE_PER_BLOCK.
+      conf.set(OZONE_SCM_CONTAINER_LAYOUT_KEY, "FILE_PER_CHUNK");
+      new KeyValueHandler(conf, context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet,
+          metrics, c -> { });
+      assertEquals(ContainerLayoutVersion.FILE_PER_BLOCK,
+          conf.getEnum(OZONE_SCM_CONTAINER_LAYOUT_KEY, ContainerLayoutVersion.FILE_PER_CHUNK));
+
       //Set a class which is not of sub class of VolumeChoosingPolicy
       conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
           "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");

diff --git a/hadoop-hdds/docs/content/feature/OM-HA.md b/hadoop-hdds/docs/content/feature/OM-HA.md
@@ -41,14 +41,6 @@ Client connects to the Leader Ozone Manager which process the request and schedu
 
 ## Configuration
 
-HA mode of Ozone Manager can be enabled with the following settings in `ozone-site.xml`:
-
-```XML
-<property>
-   <name>ozone.om.ratis.enable</name>
-   <value>true</value>
-</property>
-```
 One Ozone configuration (`ozone-site.xml`) can support multiple Ozone HA cluster. To select between the available HA clusters a logical name is required for each of the clusters which can be resolved to the IP addresses (and domain names) of the Ozone Managers.
 
 This logical name is called `serviceId` and can be configured in the `ozone-site.xml`

diff --git a/hadoop-hdds/docs/content/feature/OM-HA.zh.md b/hadoop-hdds/docs/content/feature/OM-HA.zh.md
@@ -42,15 +42,6 @@ Ozone Manager 和 Storage Container Manager 都支持 HA。在这种模式下，
 
 ## 配置
 
-可以在 `ozone-site.xml` 中配置以下设置来启用 Ozone Manager 的高可用模式：
-
-```XML
-<property>
-   <name>ozone.om.ratis.enable</name>
-   <value>true</value>
-</property>
-```
-
 一个 Ozone 的配置（`ozone-site.xml`）支持多个 Ozone 高可用集群。为了支持在多个高可用集群之间进行选择，每个集群都需要一个逻辑名称，该逻辑名称可以解析为 Ozone Manager 的 IP 地址（和域名）。
 
 该逻辑名称叫做 `serviceId`，可以在 `ozone-site.xml` 中进行配置：

diff --git a/hadoop-hdds/erasurecode/pom.xml b/hadoop-hdds/erasurecode/pom.xml
@@ -38,13 +38,13 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+
+    <!-- Test dependencies -->
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
       <scope>test</scope>
     </dependency>
-
-    <!-- Test dependencies -->
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-hadoop-dependency-test</artifactId>

diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java
@@ -139,14 +139,23 @@ public BaseHttpServer(MutableConfigurationSource conf, String name)
 
       builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
 
+      boolean addDefaultApps = shouldAddDefaultApps();
+      if (!addDefaultApps) {
+        builder.withoutDefaultApps();
+      }
+
       httpServer = builder.build();
-      httpServer.addServlet("conf", "/conf", HddsConfServlet.class);
 
-      httpServer.addServlet("logstream", "/logstream", LogStreamServlet.class);
-      prometheusSupport =
+      // TODO move these to HttpServer2.addDefaultApps
+      if (addDefaultApps) {
+        httpServer.addServlet("conf", "/conf", HddsConfServlet.class);
+        httpServer.addServlet("logstream", "/logstream", LogStreamServlet.class);
+      }
+
+      prometheusSupport = addDefaultApps &&
           conf.getBoolean(HddsConfigKeys.HDDS_PROMETHEUS_ENABLED, true);
 
-      profilerSupport =
+      profilerSupport = addDefaultApps &&
           conf.getBoolean(HddsConfigKeys.HDDS_PROFILER_ENABLED, false);
 
       if (prometheusSupport) {
@@ -477,4 +486,9 @@ public boolean isSecurityEnabled() {
 
   protected abstract String getHttpAuthConfigPrefix();
 
+  /** Override to disable the default servlets. */
+  protected boolean shouldAddDefaultApps() {
+    return true;
+  }
+
 }