From 265af96d44298c0144a551627a98fad0b2784b5b Mon Sep 17 00:00:00 2001 From: baoloongmao Date: Sun, 5 Jan 2025 11:21:14 +0800 Subject: [PATCH 1/3] Introduce LocalFileBlackHoleWriter for test flush ceiling performance --- .../impl/LocalFileBlackHoleWriter.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java diff --git a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java new file mode 100644 index 0000000000..21fec7ff5c --- /dev/null +++ b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.uniffle.storage.handler.impl; + +import io.netty.buffer.ByteBuf; +import org.apache.uniffle.storage.api.FileWriter; +import org.apache.uniffle.storage.common.FileBasedShuffleSegment; + +import java.io.File; +import java.io.IOException; + +/** A shuffle writer that write data into black hole. */ +public class LocalFileBlackHoleWriter implements FileWriter { + + private long nextOffset; + + public LocalFileBlackHoleWriter(File file, int bufferSize) {} + + @Override + public void writeData(byte[] data) throws IOException { + nextOffset = nextOffset + data.length; + } + + @Override + public void writeData(ByteBuf buf) throws IOException { + if (buf != null && buf.readableBytes() > 0) { + nextOffset = nextOffset + buf.readableBytes(); + } + } + + @Override + public void writeIndex(FileBasedShuffleSegment segment) throws IOException {} + + @Override + public long nextOffset() { + return nextOffset; + } + + @Override + public synchronized void close() throws IOException {} +} From 96ee3d7190bd71b7365ee61d69f61e35a5935e5e Mon Sep 17 00:00:00 2001 From: baoloongmao Date: Sun, 5 Jan 2025 12:25:43 +0800 Subject: [PATCH 2/3] fix spotless issue --- .../storage/handler/impl/LocalFileBlackHoleWriter.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java index 21fec7ff5c..090d4d6a2e 100644 --- a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java +++ b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java @@ -17,13 +17,14 @@ package org.apache.uniffle.storage.handler.impl; +import java.io.File; +import java.io.IOException; + import io.netty.buffer.ByteBuf; + import org.apache.uniffle.storage.api.FileWriter; import org.apache.uniffle.storage.common.FileBasedShuffleSegment; -import java.io.File; -import java.io.IOException; - /** A shuffle writer that write data into black hole. */ public class LocalFileBlackHoleWriter implements FileWriter { From 3f37afa9e59a6a582d669e0ecf037e970e000661 Mon Sep 17 00:00:00 2001 From: baoloongmao Date: Mon, 6 Jan 2025 18:33:47 +0800 Subject: [PATCH 3/3] Improve comment of LocalFileBlackHoleWriter to explain this class is used for performance test purpose only --- .../uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java index 090d4d6a2e..ba1f07c80c 100644 --- a/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java +++ b/storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileBlackHoleWriter.java @@ -25,7 +25,7 @@ import org.apache.uniffle.storage.api.FileWriter; import org.apache.uniffle.storage.common.FileBasedShuffleSegment; -/** A shuffle writer that write data into black hole. */ +/** A shuffle writer that write data into black hole for performance test purpose only. */ public class LocalFileBlackHoleWriter implements FileWriter { private long nextOffset;