From a0c63050acd9a19f4ecdbf1edac15abdb61d06cd Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Tue, 14 Jan 2025 11:40:21 -0800 Subject: [PATCH 1/7] HDDS-11784 get the parent id for MPU even it is missing parent directories --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 8 +++++--- .../S3MultipartUploadCompleteResponseWithFSO.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8f4c070b76c..2b77d9752f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -899,11 +899,13 @@ public String getMultipartKeyFSO(String volume, String bucket, String key, Strin final long volumeId = getVolumeId(volume); final long bucketId = getBucketId(volume, bucket); - long parentId = - OMFileRequest.getParentID(volumeId, bucketId, key, this); + final String nonFSOMultipartKey = + getMultipartKey(volume, bucket, key, uploadId); + final OmMultipartKeyInfo multipartKeyInfo = + getMultipartInfoTable().get(nonFSOMultipartKey); + long parentId = multipartKeyInfo.getParentID(); String fileName = OzoneFSUtils.getFileName(key); - return getMultipartKey(volumeId, bucketId, parentId, fileName, uploadId); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 4d1a6ce09bc..23d1cf85fae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -47,7 +47,7 @@ * 3) Delete unused parts. */ @CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, FILE_TABLE, DELETED_TABLE, - MULTIPARTINFO_TABLE}) + MULTIPARTINFO_TABLE, DELETED_TABLE}) public class S3MultipartUploadCompleteResponseWithFSO extends S3MultipartUploadCompleteResponse { From 719fd57dc0b7f1d6c020a4bbdc81f7596aa9a62f Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Tue, 14 Jan 2025 12:18:09 -0800 Subject: [PATCH 2/7] HDDS-11784 fix typo --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 4 ++-- .../S3MultipartUploadCompleteResponseWithFSO.java | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2b77d9752f2..eb1de11570b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -903,9 +903,9 @@ public String getMultipartKeyFSO(String volume, String bucket, String key, Strin getMultipartKey(volume, bucket, key, uploadId); final OmMultipartKeyInfo multipartKeyInfo = getMultipartInfoTable().get(nonFSOMultipartKey); - long parentId = multipartKeyInfo.getParentID(); + final long parentId = multipartKeyInfo.getParentID(); - String fileName = OzoneFSUtils.getFileName(key); + final String fileName = OzoneFSUtils.getFileName(key); return getMultipartKey(volumeId, bucketId, parentId, fileName, uploadId); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 23d1cf85fae..ff81fcd2b64 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -33,10 +33,7 @@ import java.io.IOException; import java.util.List; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.MULTIPARTINFO_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.*; /** * Response for Multipart Upload Complete request. @@ -47,7 +44,7 @@ * 3) Delete unused parts. */ @CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, FILE_TABLE, DELETED_TABLE, - MULTIPARTINFO_TABLE, DELETED_TABLE}) + MULTIPARTINFO_TABLE, DIRECTORY_TABLE}) public class S3MultipartUploadCompleteResponseWithFSO extends S3MultipartUploadCompleteResponse { From e72c6fc23eef39915966347cc63e8f8b90f06c05 Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Tue, 14 Jan 2025 12:19:01 -0800 Subject: [PATCH 3/7] HDDS-11784 fix imports --- .../multipart/S3MultipartUploadCompleteResponseWithFSO.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index ff81fcd2b64..918f8af4e00 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -33,7 +33,11 @@ import java.io.IOException; import java.util.List; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.*; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.MULTIPARTINFO_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; /** * Response for Multipart Upload Complete request. From c0629ebaa27f7002d8d2d02e6b579328c09629ff Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Tue, 14 Jan 2025 22:50:18 -0800 Subject: [PATCH 4/7] HDDS-11784 fix imports --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index eb1de11570b..968fc3e3c19 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -85,7 +85,6 @@ import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; From 9a1967bce1981d042426d33c44b05deec0ff8790 Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Sun, 19 Jan 2025 13:11:29 -0800 Subject: [PATCH 5/7] HDDS-11784 adjust getMultipartKeyFSO method --- .../ozone/om/exceptions/OMException.java | 1 + .../ozone/om/OmMetadataManagerImpl.java | 27 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 3897a022834..7da2bf68ba0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -152,6 +152,7 @@ public enum ResultCodes { NO_SUCH_MULTIPART_UPLOAD_ERROR, MISMATCH_MULTIPART_LIST, + MISSING_MULTIPART_KEY_INFO_ERROR, MISSING_UPLOAD_PARTS, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 968fc3e3c19..e2184b218ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -85,6 +85,7 @@ import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; @@ -111,10 +112,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.*; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; @@ -898,11 +897,23 @@ public String getMultipartKeyFSO(String volume, String bucket, String key, Strin final long volumeId = getVolumeId(volume); final long bucketId = getBucketId(volume, bucket); - final String nonFSOMultipartKey = - getMultipartKey(volume, bucket, key, uploadId); - final OmMultipartKeyInfo multipartKeyInfo = - getMultipartInfoTable().get(nonFSOMultipartKey); - final long parentId = multipartKeyInfo.getParentID(); + long parentId; + try { + parentId = OMFileRequest.getParentID(volumeId, bucketId, key, this); + } catch (final Exception e) { + // It is possible we miss directories and exception is thrown. + // see https://issues.apache.org/jira/browse/HDDS-11784 + LOG.warn("Got exception when finding parent id for {}/{}/{}. Use another way to get it", + volumeId, bucketId, key, e); + final String nonFSOMultipartKey = + getMultipartKey(volume, bucket, key, uploadId); + final OmMultipartKeyInfo multipartKeyInfo = + getMultipartInfoTable().get(nonFSOMultipartKey); + if (multipartKeyInfo == null) { + throw new OMException(MISSING_MULTIPART_KEY_INFO_ERROR); + } + parentId = multipartKeyInfo.getParentID(); + } final String fileName = OzoneFSUtils.getFileName(key); return getMultipartKey(volumeId, bucketId, parentId, From 2362d588b238f5080bb8b3bb756770e4a4fa6675 Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Sun, 19 Jan 2025 23:29:06 -0800 Subject: [PATCH 6/7] HDDS-11784 address PR comments --- .../ozone/om/exceptions/OMException.java | 1 - ...TestOzoneClientMultipartUploadWithFSO.java | 36 +++++++++++++++++++ .../ozone/om/OmMetadataManagerImpl.java | 7 ++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 7da2bf68ba0..3897a022834 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -152,7 +152,6 @@ public enum ResultCodes { NO_SUCH_MULTIPART_UPLOAD_ERROR, MISMATCH_MULTIPART_LIST, - MISSING_MULTIPART_KEY_INFO_ERROR, MISSING_UPLOAD_PARTS, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 58183e87705..de17e773c65 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -631,6 +631,42 @@ public void testListMultipartUploadParts() throws Exception { assertFalse(ozoneMultipartUploadPartListParts.isTruncated()); } + @Test + public void testAbortMultipartUploadSuccessWithMissingParentDirectories() throws Exception { + String parentDir = "parentDirToDelete/"; + keyName = parentDir + UUID.randomUUID(); + + OzoneManager ozoneManager = cluster.getOzoneManager(); + String buckKey = ozoneManager.getMetadataManager() + .getBucketKey(volume.getName(), bucket.getName()); + OmBucketInfo buckInfo = + ozoneManager.getMetadataManager().getBucketTable().get(buckKey); + BucketLayout bucketLayout = buckInfo.getBucketLayout(); + + String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, + ONE); + + OMMetadataManager metadataMgr = + cluster.getOzoneManager().getMetadataManager(); + String multipartOpenKey = + metadataMgr.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); + String multipartKey = metadataMgr.getMultipartKey(volumeName, bucketName, + keyName, uploadID); + + // Delete parent directory + ozClient.getProxy().deleteKey(volumeName, bucketName, parentDir, false); + + // Abort multipart upload with missing parent directory + bucket.abortMultipartUpload(keyName, uploadID); + + OmKeyInfo omKeyInfo = + metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); + OmMultipartKeyInfo omMultipartKeyInfo = + metadataMgr.getMultipartInfoTable().get(multipartKey); + assertNull(omKeyInfo); + assertNull(omMultipartKeyInfo); + } + private void verifyPartNamesInDB(Map partsMap, OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts, String uploadID) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index e2184b218ab..a77ce549e23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -905,12 +905,13 @@ public String getMultipartKeyFSO(String volume, String bucket, String key, Strin // see https://issues.apache.org/jira/browse/HDDS-11784 LOG.warn("Got exception when finding parent id for {}/{}/{}. Use another way to get it", volumeId, bucketId, key, e); - final String nonFSOMultipartKey = + final String multipartKey = getMultipartKey(volume, bucket, key, uploadId); final OmMultipartKeyInfo multipartKeyInfo = - getMultipartInfoTable().get(nonFSOMultipartKey); + getMultipartInfoTable().get(multipartKey); if (multipartKeyInfo == null) { - throw new OMException(MISSING_MULTIPART_KEY_INFO_ERROR); + LOG.error("Could not find multipartKeyInfo for {}", multipartKey); + throw new OMException(NO_SUCH_MULTIPART_UPLOAD_ERROR); } parentId = multipartKeyInfo.getParentID(); } From c42634e6910d5d25503162cb53f145d1483c7769 Mon Sep 17 00:00:00 2001 From: zhiheng xie Date: Sun, 19 Jan 2025 23:31:15 -0800 Subject: [PATCH 7/7] HDDS-11784 adjust imports --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index a77ce549e23..06bcb105842 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -112,8 +112,11 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.*; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist;