This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new a1bcc1b3a [#1164] refactor: Exposing the getDataLen method for
ShuffleDataResult (#1170)
a1bcc1b3a is described below
commit a1bcc1b3a8bf986b4b42b3c76a8c8e719a9231be
Author: pegasas <[email protected]>
AuthorDate: Mon Aug 28 09:44:01 2023 +0800
[#1164] refactor: Exposing the getDataLen method for ShuffleDataResult
(#1170)
### What changes were proposed in this pull request?
Exposing `ShuffleDataResult.getDataLen` method for being invoked by metrics
and tests.
### Why are the changes needed?
Exposing `ShuffleDataResult.getDataLen` method for being invoked by metrics
and tests.
Fix: #1164
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
No need.
---
.../main/java/org/apache/uniffle/common/ShuffleDataResult.java | 10 ++++++++++
.../org/apache/uniffle/server/ShuffleServerGrpcService.java | 4 ++--
.../apache/uniffle/server/netty/ShuffleServerNettyHandler.java | 5 ++---
.../apache/uniffle/server/buffer/ShuffleBufferManagerTest.java | 4 ++--
.../org/apache/uniffle/server/buffer/ShuffleBufferTest.java | 2 +-
.../uniffle/storage/handler/impl/LocalFileHandlerTest.java | 2 +-
6 files changed, 18 insertions(+), 9 deletions(-)
diff --git
a/common/src/main/java/org/apache/uniffle/common/ShuffleDataResult.java
b/common/src/main/java/org/apache/uniffle/common/ShuffleDataResult.java
index 35710fa81..e63357e10 100644
--- a/common/src/main/java/org/apache/uniffle/common/ShuffleDataResult.java
+++ b/common/src/main/java/org/apache/uniffle/common/ShuffleDataResult.java
@@ -71,6 +71,16 @@ public class ShuffleDataResult {
return ByteBufUtils.readBytes(buffer.byteBuf());
}
+ public int getDataLength() {
+ if (buffer == null) {
+ return 0;
+ }
+ if (buffer.nioByteBuffer().hasArray()) {
+ return buffer.nioByteBuffer().array().length;
+ }
+ return buffer.nioByteBuffer().remaining();
+ }
+
public ByteBuf getDataBuf() {
return buffer.byteBuf();
}
diff --git
a/server/src/main/java/org/apache/uniffle/server/ShuffleServerGrpcService.java
b/server/src/main/java/org/apache/uniffle/server/ShuffleServerGrpcService.java
index 21eb0ae8f..3b41af321 100644
---
a/server/src/main/java/org/apache/uniffle/server/ShuffleServerGrpcService.java
+++
b/server/src/main/java/org/apache/uniffle/server/ShuffleServerGrpcService.java
@@ -633,8 +633,8 @@ public class ShuffleServerGrpcService extends
ShuffleServerImplBase {
length);
long readTime = System.currentTimeMillis() - start;
ShuffleServerMetrics.counterTotalReadTime.inc(readTime);
-
ShuffleServerMetrics.counterTotalReadDataSize.inc(sdr.getData().length);
-
ShuffleServerMetrics.counterTotalReadLocalDataFileSize.inc(sdr.getData().length);
+ ShuffleServerMetrics.counterTotalReadDataSize.inc(sdr.getDataLength());
+
ShuffleServerMetrics.counterTotalReadLocalDataFileSize.inc(sdr.getDataLength());
shuffleServer
.getGrpcMetrics()
.recordProcessTime(ShuffleServerGrpcMetrics.GET_SHUFFLE_DATA_METHOD, readTime);
diff --git
a/server/src/main/java/org/apache/uniffle/server/netty/ShuffleServerNettyHandler.java
b/server/src/main/java/org/apache/uniffle/server/netty/ShuffleServerNettyHandler.java
index 96476c980..30a90623b 100644
---
a/server/src/main/java/org/apache/uniffle/server/netty/ShuffleServerNettyHandler.java
+++
b/server/src/main/java/org/apache/uniffle/server/netty/ShuffleServerNettyHandler.java
@@ -442,9 +442,8 @@ public class ShuffleServerNettyHandler implements
BaseMessageHandler {
length);
long readTime = System.currentTimeMillis() - start;
ShuffleServerMetrics.counterTotalReadTime.inc(readTime);
- int dataLength = sdr.getDataBuffer().remaining();
- ShuffleServerMetrics.counterTotalReadDataSize.inc(dataLength);
- ShuffleServerMetrics.counterTotalReadLocalDataFileSize.inc(dataLength);
+ ShuffleServerMetrics.counterTotalReadDataSize.inc(sdr.getDataLength());
+
ShuffleServerMetrics.counterTotalReadLocalDataFileSize.inc(sdr.getDataLength());
shuffleServer
.getNettyMetrics()
.recordProcessTime(GetLocalShuffleDataRequest.class.getName(),
readTime);
diff --git
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
index 5c7bd5eb0..a51d377e6 100644
---
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
+++
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
@@ -199,10 +199,10 @@ public class ShuffleBufferManagerTest extends
BufferTestBase {
bufferPool.get(appId).get(3).get(0).getInFlushBlockMap().clear();
// empty data return
sdr = shuffleBufferManager.getShuffleData(appId, 2, 0,
Constants.INVALID_BLOCK_ID, 60);
- assertEquals(0, sdr.getData().length);
+ assertEquals(0, sdr.getDataLength());
lastBlockId = spd2.getBlockList()[0].getBlockId();
sdr = shuffleBufferManager.getShuffleData(appId, 2, 0, lastBlockId, 100);
- assertEquals(0, sdr.getData().length);
+ assertEquals(0, sdr.getDataLength());
}
@Test
diff --git
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferTest.java
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferTest.java
index 8d5a2672d..5eb1bcfa7 100644
---
a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferTest.java
+++
b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferTest.java
@@ -310,7 +310,7 @@ public class ShuffleBufferTest extends BufferTestBase {
shuffleBuffer = new ShuffleBuffer(200);
sdr = shuffleBuffer.getShuffleData(Constants.INVALID_BLOCK_ID, 10);
assertEquals(0, sdr.getBufferSegments().size());
- assertEquals(0, sdr.getData().length);
+ assertEquals(0, sdr.getDataLength());
// case7: get data with multiple flush buffer and cached buffer
shuffleBuffer = new ShuffleBuffer(200);
diff --git
a/storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
b/storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
index 33c70027f..f6f793fb0 100644
---
a/storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
+++
b/storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
@@ -135,7 +135,7 @@ public class LocalFileHandlerTest {
targetDataFile.delete();
shuffleDataResults = LocalFileHandlerTestBase.readData(readHandler1,
shuffleIndexResult);
for (ShuffleDataResult shuffleData : shuffleDataResults) {
- assertEquals(0, shuffleData.getData().length);
+ assertEquals(0, shuffleData.getDataLength());
assertTrue(shuffleData.isEmpty());
}
}
