Merge branch 'master' into HDDS-12159

Tejaskriya · Feb 3, 2025 · 1dc25eb · 1dc25eb
2 parents 5da36ad + 24aab04
commit 1dc25eb
Show file tree

Hide file tree

Showing 212 changed files with 1,048 additions and 703 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -43,6 +43,7 @@ jobs:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
       acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
+      integration-suites: ${{ steps.integration-suites.outputs.suites }}
       needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
       needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
       basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
@@ -97,6 +98,9 @@ jobs:
       - name: Acceptance suites
         id: acceptance-suites
         run: dev-support/ci/acceptance_suites.sh
+      - name: Integration suites
+        id: integration-suites
+        run: dev-support/ci/integration_suites.sh
       - name: Categorize Basic Checks
         id: categorize-basic-checks
         env:
@@ -147,6 +151,13 @@ jobs:
         run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
+        continue-on-error: true
       - name: Store binaries for tests
         uses: actions/upload-artifact@v4
         with:
@@ -223,10 +234,17 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Compile Ozone using Java ${{ matrix.java }}
-        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
+        run: hadoop-ozone/dev-support/checks/compile.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
         env:
           OZONE_WITH_COVERAGE: false
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ${{ github.job }}-${{ matrix.java }}
+          path: target/${{ github.job }}
+        continue-on-error: true
   basic:
     needs:
       - build-info
@@ -591,17 +609,7 @@ jobs:
     if: needs.build-info.outputs.needs-integration-tests == 'true'
     strategy:
       matrix:
-        profile:
-          - client
-          - container
-          - filesystem
-          - hdds
-          - om
-          - ozone
-          - recon
-          - shell
-          - snapshot
-          - flaky
+        profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }}
       fail-fast: false
     steps:
       - name: Checkout project
@@ -639,7 +647,7 @@ jobs:
             args="$args -DskipShade"
           fi
 
-          hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
+          hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${args}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Summary of failures

diff --git a/dev-support/ci/integration_suites.sh b/dev-support/ci/integration_suites.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# shellcheck source=dev-support/ci/lib/_script_init.sh
+. dev-support/ci/lib/_script_init.sh
+
+# output test suites without test- prefix
+SUITES=$(grep -o '<id>test-[^<]\+' pom.xml \
+  | sort -u | cut -f2 -d'-')
+
+initialization::ga_output suites \
+    "$(initialization::parameters_to_json ${SUITES})"
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
@@ -54,12 +54,10 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>commons-collections</groupId>
@@ -84,8 +82,6 @@
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-api</artifactId>
-      <version>${io.grpc.version}</version>
-      <scope>compile</scope>
       <exclusions>
         <exclusion>
           <groupId>com.google.code.findbugs</groupId>
@@ -186,7 +182,6 @@
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcpkix-jdk18on</artifactId>
-      <version>${bouncycastle.version}</version>
     </dependency>
     <dependency>
       <groupId>org.bouncycastle</groupId>
@@ -333,7 +328,6 @@
       <extension>
         <groupId>kr.motd.maven</groupId>
         <artifactId>os-maven-plugin</artifactId>
-        <version>${os-maven-plugin.version}</version>
       </extension>
     </extensions>
   </build>

diff --git a/...apache/hadoop/hdds/cli/AbstractMixin.java → ...apache/hadoop/hdds/cli/AbstractMixin.java b/...apache/hadoop/hdds/cli/AbstractMixin.java → ...apache/hadoop/hdds/cli/AbstractMixin.java
diff --git a/...e/hadoop/hdds/cli/AbstractSubcommand.java → ...e/hadoop/hdds/cli/AbstractSubcommand.java b/...e/hadoop/hdds/cli/AbstractSubcommand.java → ...e/hadoop/hdds/cli/AbstractSubcommand.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java
@@ -26,6 +26,7 @@
 public enum ReplicationType {
   RATIS,
   STAND_ALONE,
+  @Deprecated
   CHAINED,
   EC;
 

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java
@@ -82,5 +82,10 @@ public long getUsedSpace() {
     public SpaceUsageSource snapshot() {
       return this; // immutable
     }
+
+    @Override
+    public String toString() {
+      return "capacity=" + capacity + ", used=" + used + ", available=" + available;
+    }
   }
 }
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1369,9 +1369,8 @@
       specified when writing the key. The default is used if replication
       is not specified when creating key or no default replication set at
       bucket. Supported values:
-      For Standalone: 1
-      For Ratis: 3
-      For Erasure Coding(EC) supported format:
+      For RATIS: 1, 3
+      For EC (Erasure Coding) supported format:
       {ECCodec}-{DataBlocks}-{ParityBlocks}-{ChunkSize}
         ECCodec: Codec for encoding stripe. Supported values : XOR, RS (Reed Solomon)
         DataBlocks: Number of data blocks in a stripe.
@@ -1389,7 +1388,7 @@
       Default replication type to be used while writing key into ozone. The
       value can be specified when writing the key, default is used when
       nothing is specified when creating key or no default value set at bucket.
-      Supported values: RATIS, STAND_ALONE, CHAINED and EC.
+      Supported values: RATIS, EC.
     </description>
   </property>
 
@@ -3687,14 +3686,6 @@
       be defined with postfix (ns,ms,s,m,h,d)
     </description>
   </property>
-  <property>
-    <name>ozone.path.deleting.limit.per.task</name>
-    <value>6000</value>
-    <tag>OZONE, PERFORMANCE, OM</tag>
-    <description>A maximum number of paths(dirs/files) to be deleted by
-      directory deleting service per time interval.
-    </description>
-  </property>
   <property>
     <name>ozone.snapshot.filtering.limit.per.task</name>
     <value>2</value>

diff --git a/...r-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/...r-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -622,12 +622,11 @@ private boolean isVolumeFull(Container container) {
         .orElse(Boolean.FALSE);
     if (isOpen) {
       HddsVolume volume = container.getContainerData().getVolume();
-      SpaceUsageSource precomputedVolumeSpace =
-          volume.getCurrentUsage();
-      long volumeCapacity = precomputedVolumeSpace.getCapacity();
+      SpaceUsageSource usage = volume.getCurrentUsage();
+      long volumeCapacity = usage.getCapacity();
       long volumeFreeSpaceToSpare =
           freeSpaceCalculator.get(volumeCapacity);
-      long volumeFree = precomputedVolumeSpace.getAvailable();
+      long volumeFree = usage.getAvailable();
       long volumeCommitted = volume.getCommittedBytes();
       long volumeAvailable = volumeFree - volumeCommitted;
       return (volumeAvailable <= volumeFreeSpaceToSpare);

diff --git a/...e/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java b/...e/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.ozone.container.common.volume;
 
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
+
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Predicate;
@@ -38,8 +40,9 @@ public class AvailableSpaceFilter implements Predicate<HddsVolume> {
 
   @Override
   public boolean test(HddsVolume vol) {
-    long volumeCapacity = vol.getCapacity();
-    long free = vol.getAvailable();
+    SpaceUsageSource usage = vol.getCurrentUsage();
+    long volumeCapacity = usage.getCapacity();
+    long free = usage.getAvailable();
     long committed = vol.getCommittedBytes();
     long available = free - committed;
     long volumeFreeSpaceToSpare =

diff --git a/...in/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java b/...in/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
@@ -91,9 +91,9 @@ public HddsVolume chooseVolume(List<HddsVolume> volumes,
       HddsVolume firstVolume = volumesWithEnoughSpace.get(firstIndex);
       HddsVolume secondVolume = volumesWithEnoughSpace.get(secondIndex);
 
-      long firstAvailable = firstVolume.getAvailable()
+      long firstAvailable = firstVolume.getCurrentUsage().getAvailable()
           - firstVolume.getCommittedBytes();
-      long secondAvailable = secondVolume.getAvailable()
+      long secondAvailable = secondVolume.getCurrentUsage().getAvailable()
           - secondVolume.getCommittedBytes();
       return firstAvailable < secondAvailable ? secondVolume : firstVolume;
     }

diff --git a/...ainer-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java b/...ainer-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
@@ -57,9 +57,9 @@ protected DbVolume(Builder b) throws IOException {
 
     this.hddsDbStorePathMap = new HashMap<>();
     if (!b.getFailedVolume() && getVolumeInfo().isPresent()) {
-      LOG.info("Creating DbVolume: {} of storage type : {} capacity : {}",
+      LOG.info("Creating DbVolume: {} of storage type: {}, {}",
               getStorageDir(), b.getStorageType(),
-              getVolumeInfo().get().getCapacity());
+              getCurrentUsage());
 
       initialize();
     }

diff --git a/...ner-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/...ner-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -131,9 +131,10 @@ private HddsVolume(Builder b) throws IOException {
       this.volumeInfoMetrics =
           new VolumeInfoMetrics(b.getVolumeRootStr(), this);
 
-      LOG.info("Creating HddsVolume: {} of storage type : {} capacity : {}",
-          getStorageDir(), b.getStorageType(),
-              getVolumeInfo().get().getCapacity());
+      LOG.info("Creating HddsVolume: {} of storage type: {}, {}",
+          getStorageDir(),
+          b.getStorageType(),
+          getCurrentUsage());
 
       initialize();
     } else {

diff --git a/...rvice/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/...rvice/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
@@ -479,9 +480,10 @@ public StorageLocationReport[] getStorageReport() {
         if (volumeInfo.isPresent()) {
           try {
             rootDir = volumeInfo.get().getRootDir();
-            scmUsed = volumeInfo.get().getScmUsed();
-            remaining = volumeInfo.get().getAvailable();
-            capacity = volumeInfo.get().getCapacity();
+            SpaceUsageSource usage = volumeInfo.get().getCurrentUsage();
+            scmUsed = usage.getUsedSpace();
+            remaining = usage.getAvailable();
+            capacity = usage.getCapacity();
             committed = (volume instanceof HddsVolume) ?
                 ((HddsVolume) volume).getCommittedBytes() : 0;
             failed = false;

diff --git a/...-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/...-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -447,25 +447,11 @@ public String getVolumeRootDir() {
     return volumeInfo.map(VolumeInfo::getRootDir).orElse(null);
   }
 
-  public long getCapacity() {
-    return volumeInfo.map(VolumeInfo::getCapacity).orElse(0L);
-  }
-
-  public long getAvailable() {
-    return volumeInfo.map(VolumeInfo::getAvailable).orElse(0L);
-
-  }
-
   public SpaceUsageSource getCurrentUsage() {
     return volumeInfo.map(VolumeInfo::getCurrentUsage)
         .orElse(SpaceUsageSource.UNKNOWN);
   }
 
-  public long getUsedSpace() {
-    return volumeInfo.map(VolumeInfo::getScmUsed).orElse(0L);
-
-  }
-
   public File getStorageDir() {
     return this.storageDir;
   }

diff --git a/...ner-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/...ner-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -152,24 +152,6 @@ private VolumeInfo(Builder b) throws IOException {
     usage = new VolumeUsage(checkParams, b.conf);
   }
 
-  public long getCapacity() {
-    return usage.getCapacity();
-  }
-
-  /**
-   * <pre>
-   * {@code
-   * Calculate available space use method A.
-   * |----used----|   (avail)   |++++++++reserved++++++++|
-   * |<-     capacity         ->|
-   * A) avail = capacity - used
-   * }
-   * </pre>
-   */
-  public long getAvailable() {
-    return usage.getAvailable();
-  }
-
   public SpaceUsageSource getCurrentUsage() {
     return usage.getCurrentUsage();
   }
@@ -186,10 +168,6 @@ public void refreshNow() {
     usage.refreshNow();
   }
 
-  public long getScmUsed() {
-    return usage.getUsedSpace();
-  }
-
   void shutdownUsageThread() {
     usage.shutdown();
   }