This is an automated email from the ASF dual-hosted git repository.
biyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 27cdede87 [core] Get size should after close output in emptyIndexFile
(#3935)
27cdede87 is described below
commit 27cdede8708b1c3481e4972a7786b94c14c6dee6
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Aug 12 15:12:13 2024 +0800
[core] Get size should after close output in emptyIndexFile (#3935)
---
.../DeletionVectorIndexFileWriter.java | 30 ++++++++--------------
.../DeletionVectorsIndexFileTest.java | 2 +-
2 files changed, 12 insertions(+), 20 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
index 4dddd6ba1..a1d85b5ba 100644
---
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
@@ -72,9 +72,8 @@ public class DeletionVectorIndexFileWriter {
try {
while (iterator.hasNext()) {
Map.Entry<String, DeletionVector> entry = iterator.next();
- long currentSize = writer.write(entry.getKey(),
entry.getValue());
-
- if (writer.writtenSizeInBytes() + currentSize >
targetSizeInBytes) {
+ writer.write(entry.getKey(), entry.getValue());
+ if (writer.writtenSizeInBytes() > targetSizeInBytes) {
break;
}
}
@@ -92,50 +91,43 @@ public class DeletionVectorIndexFileWriter {
* <p>TODO: We can consider sending a message to delete the deletion file
in the future.
*/
private List<IndexFileMeta> emptyIndexFile() throws IOException {
- try (SingleIndexFileWriter writer = new SingleIndexFileWriter()) {
- return Collections.singletonList(writer.writtenIndexFile());
- }
+ SingleIndexFileWriter writer = new SingleIndexFileWriter();
+ writer.close();
+ return Collections.singletonList(writer.writtenIndexFile());
}
private class SingleIndexFileWriter implements Closeable {
private final Path path;
-
private final DataOutputStream dataOutputStream;
-
private final LinkedHashMap<String, Pair<Integer, Integer>> dvRanges;
- private long writtenSizeInBytes = 0L;
-
- public SingleIndexFileWriter() throws IOException {
+ private SingleIndexFileWriter() throws IOException {
this.path = indexPathFactory.newPath();
this.dataOutputStream = new
DataOutputStream(fileIO.newOutputStream(path, true));
dataOutputStream.writeByte(VERSION_ID_V1);
this.dvRanges = new LinkedHashMap<>();
}
- public long writtenSizeInBytes() {
- return this.writtenSizeInBytes;
+ private long writtenSizeInBytes() {
+ return dataOutputStream.size();
}
- public long write(String key, DeletionVector deletionVector) throws
IOException {
+ private void write(String key, DeletionVector deletionVector) throws
IOException {
Preconditions.checkNotNull(dataOutputStream);
byte[] data = deletionVector.serializeToBytes();
int size = data.length;
-
dvRanges.put(key, Pair.of(dataOutputStream.size(), size));
dataOutputStream.writeInt(size);
dataOutputStream.write(data);
dataOutputStream.writeInt(calculateChecksum(data));
- writtenSizeInBytes += size;
- return size;
}
- public IndexFileMeta writtenIndexFile() throws IOException {
+ public IndexFileMeta writtenIndexFile() {
return new IndexFileMeta(
DELETION_VECTORS_INDEX,
path.getName(),
- fileIO.getFileSize(path),
+ writtenSizeInBytes(),
dvRanges.size(),
dvRanges);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
index 2b1c98550..a4dd8f92c 100644
---
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
@@ -160,7 +160,7 @@ public class DeletionVectorsIndexFileTest {
List<IndexFileMeta> indexFiles =
deletionVectorsIndexFile.write(fileToDV);
// assert 1
- assertThat(indexFiles.size()).isEqualTo(5);
+ assertThat(indexFiles.size()).isEqualTo(3);
Map<String, DeletionVector> dvs =
deletionVectorsIndexFile.readAllDeletionVectors(indexFiles);
for (String file : dvs.keySet()) {
