This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-paimon-webui.git
The following commit(s) were added to refs/heads/main by this push:
new 11842e3 [Improvement] Enhance alignment of metadata fields (#186)
11842e3 is described below
commit 11842e3efca3981b18a0df97ddc09cd77fbff007
Author: s7monk <[email protected]>
AuthorDate: Tue Feb 6 14:14:51 2024 +0800
[Improvement] Enhance alignment of metadata fields (#186)
---
.../paimon/web/server/data/vo/DataFileVO.java | 26 ++++++++++++++
.../paimon/web/server/data/vo/ManifestsVO.java | 4 +++
.../paimon/web/server/data/vo/SnapshotVO.java | 25 +++++++++++++
.../server/service/impl/MetadataServiceImpl.java | 42 ++++++++++++++++++----
.../server/controller/MetadataControllerTest.java | 39 ++++++++++++++++++++
5 files changed, 130 insertions(+), 6 deletions(-)
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
index 5017325..a1ca63d 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/DataFileVO.java
@@ -23,6 +23,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
+import java.time.LocalDateTime;
+
/** VO of metadata data file. */
@Data
@Builder
@@ -37,4 +39,28 @@ public class DataFileVO {
private String filePath;
private String fileFormat;
+
+ private Long schemaId;
+
+ private Integer level;
+
+ private Long recordCount;
+
+ private Long fileSizeInBytes;
+
+ private String minKey;
+
+ private String maxKey;
+
+ private String nullValueCounts;
+
+ private String minValueStats;
+
+ private String maxValueStats;
+
+ private Long minSequenceNumber;
+
+ private Long maxSequenceNumber;
+
+ private LocalDateTime creationTime;
}
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
index bfab103..2bd4d72 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/ManifestsVO.java
@@ -35,4 +35,8 @@ public class ManifestsVO {
private Long fileSize;
private Long numAddedFiles;
+
+ private Long numDeletedFiles;
+
+ private Long schemaId;
}
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SnapshotVO.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SnapshotVO.java
index 6040426..38d2ff1 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SnapshotVO.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/data/vo/SnapshotVO.java
@@ -31,9 +31,34 @@ import java.time.LocalDateTime;
@NoArgsConstructor
@AllArgsConstructor
public class SnapshotVO {
+
private Long snapshotId;
+
private Long schemaId;
+
+ private String commitUser;
+
private Long commitIdentifier;
+
private String commitKind;
+
private LocalDateTime commitTime;
+
+ private String baseManifestList;
+
+ private String deltaManifestList;
+
+ private String changelogManifestList;
+
+ private Long totalRecordCount;
+
+ private Long deltaRecordCount;
+
+ private Long changelogRecordCount;
+
+ private Integer addedFileCount;
+
+ private Integer deletedFileCount;
+
+ private Long watermark;
}
diff --git
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
index 68faba0..9677338 100644
---
a/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
+++
b/paimon-web-server/src/main/java/org/apache/paimon/web/server/service/impl/MetadataServiceImpl.java
@@ -110,9 +110,19 @@ public class MetadataServiceImpl implements
MetadataService {
SnapshotVO.builder()
.snapshotId(internalRow.getLong(0))
.schemaId(internalRow.getLong(1))
+ .commitUser(getSafeString(internalRow,
2))
.commitIdentifier(internalRow.getLong(3))
.commitKind(getSafeString(internalRow,
4))
.commitTime(getSafeLocalDateTime(internalRow, 5))
+
.baseManifestList(getSafeString(internalRow, 6))
+
.deltaManifestList(getSafeString(internalRow, 7))
+
.changelogManifestList(getSafeString(internalRow, 8))
+
.totalRecordCount(internalRow.getLong(9))
+
.deltaRecordCount(internalRow.getLong(10))
+
.changelogRecordCount(internalRow.getLong(11))
+ .addedFileCount(internalRow.getInt(12))
+
.deletedFileCount(internalRow.getInt(13))
+ .watermark(getSafeLong(internalRow,
14))
.build();
result.add(build);
});
@@ -137,6 +147,8 @@ public class MetadataServiceImpl implements MetadataService
{
.fileName(getSafeString(internalRow,
0))
.fileSize(internalRow.getLong(1))
.numAddedFiles(internalRow.getLong(2))
+
.numDeletedFiles(internalRow.getLong(3))
+ .schemaId(internalRow.getLong(4))
.build();
result.add(manifestsVo);
});
@@ -157,12 +169,26 @@ public class MetadataServiceImpl implements
MetadataService {
try {
reader.forEachRemaining(
internalRow -> {
- DataFileVO dataFileVo = new DataFileVO();
- dataFileVo.setPartition(getSafeString(internalRow, 0));
- dataFileVo.setBucket(internalRow.getInt(1));
- dataFileVo.setFilePath(getSafeString(internalRow, 2));
- dataFileVo.setFileFormat(getSafeString(internalRow,
3));
- result.add(dataFileVo);
+ DataFileVO dataFileVO =
+ DataFileVO.builder()
+ .partition(getSafeString(internalRow,
0))
+ .bucket(internalRow.getInt(1))
+ .filePath(getSafeString(internalRow,
2))
+ .fileFormat(getSafeString(internalRow,
3))
+ .schemaId(internalRow.getLong(4))
+ .level(internalRow.getInt(5))
+ .recordCount(internalRow.getLong(6))
+
.fileSizeInBytes(internalRow.getLong(7))
+ .minKey(getSafeString(internalRow, 8))
+ .maxKey(getSafeString(internalRow, 9))
+
.nullValueCounts(getSafeString(internalRow, 10))
+
.minValueStats(getSafeString(internalRow, 11))
+
.maxValueStats(getSafeString(internalRow, 12))
+
.minSequenceNumber(internalRow.getLong(13))
+
.maxSequenceNumber(internalRow.getLong(14))
+
.creationTime(getSafeLocalDateTime(internalRow, 15))
+ .build();
+ result.add(dataFileVO);
});
} catch (IOException e) {
throw new RuntimeException(e);
@@ -231,6 +257,10 @@ public class MetadataServiceImpl implements
MetadataService {
return internalRow.isNullAt(index) ? "" :
internalRow.getString(index).toString();
}
+ private Long getSafeLong(InternalRow internalRow, int index) {
+ return internalRow.isNullAt(index) ? null : internalRow.getLong(14);
+ }
+
private LocalDateTime getSafeLocalDateTime(InternalRow internalRow, int
index) {
return internalRow.isNullAt(index)
? null
diff --git
a/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
b/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
index e07132e..dc0a813 100644
---
a/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
+++
b/paimon-web-server/src/test/java/org/apache/paimon/web/server/controller/MetadataControllerTest.java
@@ -59,6 +59,7 @@ import
org.springframework.test.web.servlet.result.MockMvcResultHandlers;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -67,6 +68,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Tests for {@link MetadataController}. */
@@ -354,6 +356,8 @@ public class MetadataControllerTest extends
ControllerTestBase {
assertNotNull(manifestsVO.getFileName());
assertTrue(manifestsVO.getFileSize() > 0);
assertEquals(3, manifestsVO.getNumAddedFiles());
+ assertEquals(0, manifestsVO.getNumDeletedFiles());
+ assertEquals(0, manifestsVO.getSchemaId());
}
@Test
@@ -385,10 +389,34 @@ public class MetadataControllerTest extends
ControllerTestBase {
// Make assertions on each field of the DataFileVO class.
DataFileVO dataFileVO = dataFileVOS.get(0);
+ CommitMessageImpl commitMessage = (CommitMessageImpl) messages.get(0);
assertEquals("[2023-12-04 00:00:00]", dataFileVO.getPartition());
assertEquals(0, dataFileVO.getBucket());
assertNotNull(dataFileVO.getFilePath());
assertEquals("orc", dataFileVO.getFileFormat());
+ assertEquals(0, dataFileVO.getLevel());
+ assertEquals(1, dataFileVO.getRecordCount());
+ assertEquals(
+ commitMessage.newFilesIncrement().newFiles().get(0).fileSize(),
+ dataFileVO.getFileSizeInBytes());
+ assertEquals("[1]", dataFileVO.getMinKey());
+ assertEquals("[1]", dataFileVO.getMaxKey());
+ assertEquals("{age=0, create_time=0, id=0, name=0}",
dataFileVO.getNullValueCounts());
+ assertEquals(
+ "{age=24, create_time=2023-12-04 00:00, id=1, name=Alice}",
+ dataFileVO.getMinValueStats());
+ assertEquals(
+ "{age=24, create_time=2023-12-04 00:01, id=1, name=Alice}",
+ dataFileVO.getMaxValueStats());
+ assertEquals(0, dataFileVO.getMinSequenceNumber());
+ assertEquals(0, dataFileVO.getMaxSequenceNumber());
+ assertEquals(
+
commitMessage.newFilesIncrement().newFiles().get(0).creationTimeEpochMillis(),
+ dataFileVO
+ .getCreationTime()
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli());
List<String> actualPartitions = new ArrayList<>();
List<String> expectedPartitions =
@@ -439,9 +467,20 @@ public class MetadataControllerTest extends
ControllerTestBase {
SnapshotVO snapshotVO = snapshotVOS.get(0);
assertEquals(1, snapshotVO.getSnapshotId());
assertEquals(0, snapshotVO.getSchemaId());
+ assertFalse(snapshotVO.getCommitUser().isEmpty());
assertEquals("OVERWRITE", snapshotVO.getCommitKind());
assertTrue(snapshotVO.getCommitIdentifier() > 0);
assertTrue(snapshotVO.getCommitTime().isBefore(LocalDateTime.now()));
+ assertEquals(0, snapshotVO.getSchemaId());
+ assertFalse(snapshotVO.getDeltaManifestList().isEmpty());
+ assertFalse(snapshotVO.getBaseManifestList().isEmpty());
+ assertEquals("", snapshotVO.getChangelogManifestList());
+ assertEquals(3, snapshotVO.getTotalRecordCount());
+ assertEquals(3, snapshotVO.getDeltaRecordCount());
+ assertEquals(0, snapshotVO.getChangelogRecordCount());
+ assertEquals(3, snapshotVO.getAddedFileCount());
+ assertEquals(0, snapshotVO.getDeletedFileCount());
+ assertNull(snapshotVO.getWatermark());
}
@Test
