This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 66b9c6eeb60f fix(trino): report real block size and slice splits
solely by target_split_size (#19478)
66b9c6eeb60f is described below
commit 66b9c6eeb60f377b86ebaaa7a0e96d37f007593b
Author: voonhous <[email protected]>
AuthorDate: Tue Aug 4 19:08:57 2026 +0800
fix(trino): report real block size and slice splits solely by
target_split_size (#19478)
* fix(trino): report real block size in HudiTrinoStorage and slice splits
by target_split_size
HudiTrinoStorage hardcoded blockSize=0 in convertToPathInfo and getPathInfo,
the hudi-trino counterpart of trinodb/trino#29842. Report the file length as
the block size instead, matching the upstream fix at the storage layer.
HudiSplitFactory used max(target_split_size, blockSize) to size base file
splits. With the storage layer now reporting length as block size, that
max()
would silently disable the target_split_size knob, so make the policy
explicit: slicing is governed solely by target_split_size. Also fail fast on
a non-positive target, which previously looped forever.
Covers apache/hudi#19231.
* fix(trino): validate target_split_size at config time and pin the split
sizing tests
Addresses review feedback on #19478:
- Move the non-positive target split size guard from the middle of
createSplitsForBaseFile into the HudiSplitFactory constructor, so it
covers
every split path instead of only base file slicing past the fileSize == 0
early return. createHudiSplits becomes private, leaving the constructor as
the single entry point that can be handed a bad target.
- Reject a zero target at config time as well: @MinDataSize("1B") on
HudiConfig.getTargetSplitSize and validateMinDataSize on the
target_split_size session property, matching the shape already used by
parquet_small_file_threshold. The constructor check stays as a backstop.
- Set the TestHudiSplitFactory fixture block size to the base file length,
which is what HudiTrinoStorage now reports. The old fixed 8MB fixture was
below the 128MB target, so the default-target tests passed on master
unchanged and did not pin the max() removal. With the block size tracking
the file length, restoring max(target, blockSize) fails 5 tests, including
the 500MB default-target case.
---
.../main/java/io/trino/plugin/hudi/HudiConfig.java | 2 +
.../trino/plugin/hudi/HudiSessionProperties.java | 2 +
.../trino/plugin/hudi/split/HudiSplitFactory.java | 8 +-
.../plugin/hudi/storage/HudiTrinoStorage.java | 5 +-
.../java/io/trino/plugin/hudi/TestHudiConfig.java | 13 ++
.../plugin/hudi/TestHudiSessionProperties.java | 18 +++
.../plugin/hudi/split/TestHudiSplitFactory.java | 78 ++++++++---
.../plugin/hudi/storage/TestHudiTrinoStorage.java | 150 +++++++++++++++++++++
8 files changed, 257 insertions(+), 19 deletions(-)
diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java
index 8ecb76eeda9d..c5ce5ce0e23d 100644
--- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java
+++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiConfig.java
@@ -19,6 +19,7 @@ import io.airlift.configuration.ConfigDescription;
import io.airlift.configuration.DefunctConfig;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
+import io.airlift.units.MinDataSize;
import jakarta.validation.constraints.DecimalMax;
import jakarta.validation.constraints.DecimalMin;
import jakarta.validation.constraints.Min;
@@ -220,6 +221,7 @@ public class HudiConfig
}
@NotNull
+ @MinDataSize("1B")
public DataSize getTargetSplitSize()
{
return targetSplitSize;
diff --git
a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
index 32ac46a95977..8a47e3a8b03d 100644
--- a/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
+++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
@@ -31,6 +31,7 @@ import static
com.google.common.collect.ImmutableList.toImmutableList;
import static
io.trino.plugin.base.session.PropertyMetadataUtil.dataSizeProperty;
import static
io.trino.plugin.base.session.PropertyMetadataUtil.durationProperty;
import static
io.trino.plugin.base.session.PropertyMetadataUtil.validateMaxDataSize;
+import static
io.trino.plugin.base.session.PropertyMetadataUtil.validateMinDataSize;
import static
io.trino.plugin.hive.parquet.ParquetReaderConfig.PARQUET_READER_MAX_SMALL_FILE_THRESHOLD;
import static io.trino.spi.StandardErrorCode.INVALID_SESSION_PROPERTY;
import static io.trino.spi.session.PropertyMetadata.booleanProperty;
@@ -188,6 +189,7 @@ public class HudiSessionProperties
TARGET_SPLIT_SIZE,
"The target split size",
hudiConfig.getTargetSplitSize(),
+ value -> validateMinDataSize(TARGET_SPLIT_SIZE, value,
DataSize.ofBytes(1)),
false),
integerProperty(
MAX_SPLITS_PER_SECOND,
diff --git
a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
index 6bab996e199b..ece6472473c0 100644
--- a/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
+++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
@@ -51,6 +51,8 @@ public class HudiSplitFactory
this.hudiTableHandle = requireNonNull(hudiTableHandle,
"hudiTableHandle is null");
this.hudiSplitWeightProvider = requireNonNull(hudiSplitWeightProvider,
"hudiSplitWeightProvider is null");
this.targetSplitSize = requireNonNull(targetSplitSize,
"targetSplitSize is null");
+ // A non-positive target would make split generation loop forever, so
reject it here rather than mid-scan
+ checkArgument(targetSplitSize.toBytes() > 0, "targetSplitSize must be
positive: %s", targetSplitSize);
}
public List<HudiSplit> createSplits(List<HivePartitionKey> partitionKeys,
FileSlice fileSlice, String commitTime)
@@ -65,7 +67,7 @@ public class HudiSplitFactory
* <p>
* For regular MOR tables, a single split is created for the combination
of the base file and its log files.
*/
- public static List<HudiSplit> createHudiSplits(
+ private static List<HudiSplit> createHudiSplits(
HudiTableHandle hudiTableHandle,
List<HivePartitionKey> partitionKeys,
FileSlice fileSlice,
@@ -127,7 +129,9 @@ public class HudiSplitFactory
}
ImmutableList.Builder<HudiSplit> splits = ImmutableList.builder();
- long targetSplitSizeInBytes = Math.max(targetSplitSize.toBytes(),
baseFile.getPathInfo().getBlockSize());
+ // Slicing is governed solely by the target split size; the block size
reported by
+ // storage is not meaningful on object stores and must not influence
split sizing.
+ long targetSplitSizeInBytes = targetSplitSize.toBytes();
long bytesRemaining = fileSize;
while (((double) bytesRemaining) / targetSplitSizeInBytes >
SPLIT_SLOP) {
diff --git
a/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
index 48c5409c10d8..1edff63e8a7b 100644
---
a/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
+++
b/hudi-trino/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
@@ -71,7 +71,7 @@ public class HudiTrinoStorage
fileEntry.length(),
false,
(short) 0,
- 0,
+ fileEntry.length(),
fileEntry.lastModified().toEpochMilli());
}
@@ -170,7 +170,8 @@ public class HudiTrinoStorage
if (!inputFile.exists()) {
throw new FileNotFoundException("Path " + path + " does not
exist");
}
- return new StoragePathInfo(path, inputFile.length(), false, (short) 0,
0, inputFile.lastModified().toEpochMilli());
+ long length = inputFile.length();
+ return new StoragePathInfo(path, length, false, (short) 0, length,
inputFile.lastModified().toEpochMilli());
}
@Override
diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java
b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java
index d5353b896a1a..29aeded55ffd 100644
--- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java
+++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiConfig.java
@@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
+import io.airlift.units.MinDataSize;
import org.junit.jupiter.api.Test;
import java.util.Map;
@@ -24,6 +25,7 @@ import java.util.Map;
import static
io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static
io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
+import static io.airlift.testing.ValidationAssertions.assertFailsValidation;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
public class TestHudiConfig
@@ -131,4 +133,15 @@ public class TestHudiConfig
assertFullMapping(properties, expected);
}
+
+ @Test
+ public void testTargetSplitSizeValidation()
+ {
+ // A zero target split size would make split generation loop forever,
so reject it at config time
+ assertFailsValidation(
+ new HudiConfig().setTargetSplitSize(DataSize.ofBytes(0)),
+ "targetSplitSize",
+ "must be greater than or equal to 1B",
+ MinDataSize.class);
+ }
}
diff --git
a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java
b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java
index ed17a203089e..b97bdc47750e 100644
---
a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java
+++
b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiSessionProperties.java
@@ -14,14 +14,18 @@
package io.trino.plugin.hudi;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import io.trino.plugin.hive.parquet.ParquetReaderConfig;
+import io.trino.spi.TrinoException;
import io.trino.spi.connector.ConnectorSession;
import io.trino.testing.TestingConnectorSession;
import org.junit.jupiter.api.Test;
import static io.trino.plugin.hudi.HudiSessionProperties.getColumnsToHide;
import static io.trino.plugin.hudi.HudiSessionProperties.getRecordMergerImpls;
+import static io.trino.plugin.hudi.HudiSessionProperties.getTargetSplitSize;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class TestHudiSessionProperties
{
@@ -50,4 +54,18 @@ public class TestHudiSessionProperties
assertThat(getRecordMergerImpls(session))
.containsExactly("com.example.MergerOne",
"com.example.MergerTwo");
}
+
+ @Test
+ public void testSessionPropertyTargetSplitSizeRejectsZero()
+ {
+ // A zero target split size would make split generation loop forever,
so reject it when the property is read
+ HudiSessionProperties sessionProperties = new
HudiSessionProperties(new HudiConfig(), new ParquetReaderConfig());
+ ConnectorSession session = TestingConnectorSession.builder()
+ .setPropertyMetadata(sessionProperties.getSessionProperties())
+ .setPropertyValues(ImmutableMap.of("target_split_size", "0B"))
+ .build();
+ assertThatThrownBy(() -> getTargetSplitSize(session))
+ .isInstanceOf(TrinoException.class)
+ .hasMessageContaining("target_split_size must be at least 1B:
0B");
+ }
}
diff --git
a/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java
b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java
index d659044d2456..067c0f7b3dbd 100644
---
a/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java
+++
b/hudi-trino/src/test/java/io/trino/plugin/hudi/split/TestHudiSplitFactory.java
@@ -35,6 +35,7 @@ import java.util.OptionalLong;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class TestHudiSplitFactory
{
@@ -101,19 +102,63 @@ public class TestHudiSplitFactory
}
@Test
- public void testCreateHudiSplitsWithLargerBlockSize()
+ public void testCreateHudiSplitsIgnoresBlockSize()
{
- // Test with 1MB target split size and 32MB base file
- // - should create 4 splits because the block size of 8MB is larger
than the target split size
+ // Test with 2MB target and 8MB base file whose reported block size is
8MB
+ // - the block size must be ignored, so 4 splits of the 2MB target
size are expected
+ // (previously the 8MB block size beat the target and produced 1
split of 8MB)
testSplitCreation(
- DataSize.of(1, MEGABYTE),
- DataSize.of(32, MEGABYTE),
+ DataSize.of(2, MEGABYTE),
+ DataSize.of(8, MEGABYTE),
Option.empty(),
ImmutableList.of(
- Pair.of(0L, DataSize.of(8, MEGABYTE)),
- Pair.of(DataSize.of(8, MEGABYTE).toBytes(),
DataSize.of(8, MEGABYTE)),
- Pair.of(DataSize.of(16, MEGABYTE).toBytes(),
DataSize.of(8, MEGABYTE)),
- Pair.of(DataSize.of(24, MEGABYTE).toBytes(),
DataSize.of(8, MEGABYTE))));
+ Pair.of(0L, DataSize.of(2, MEGABYTE)),
+ Pair.of(DataSize.of(2, MEGABYTE).toBytes(),
DataSize.of(2, MEGABYTE)),
+ Pair.of(DataSize.of(4, MEGABYTE).toBytes(),
DataSize.of(2, MEGABYTE)),
+ Pair.of(DataSize.of(6, MEGABYTE).toBytes(),
DataSize.of(2, MEGABYTE))));
+ }
+
+ @Test
+ public void testCreateHudiSplitsWithFileSmallerThanDefaultTarget()
+ {
+ // Regression test for the split inflation reported in
trinodb/trino#29842 (hudi#19231):
+ // a ~120MB file with the default 128MB target must produce exactly 1
split
+ testSplitCreation(
+ DataSize.of(128, MEGABYTE),
+ DataSize.of(120, MEGABYTE),
+ Option.empty(),
+ ImmutableList.of(
+ Pair.of(0L, DataSize.of(120, MEGABYTE))));
+ }
+
+ @Test
+ public void testCreateHudiSplitsWithFileLargerThanDefaultTarget()
+ {
+ // Test with 128MB target and 500MB base file
+ // - should be sliced at target boundaries into 3 x 128MB + 116MB
remainder, even though the
+ // reported block size (500MB, the file length) would otherwise
force a single split
+ testSplitCreation(
+ DataSize.of(128, MEGABYTE),
+ DataSize.of(500, MEGABYTE),
+ Option.empty(),
+ ImmutableList.of(
+ Pair.of(0L, DataSize.of(128, MEGABYTE)),
+ Pair.of(DataSize.of(128, MEGABYTE).toBytes(),
DataSize.of(128, MEGABYTE)),
+ Pair.of(DataSize.of(256, MEGABYTE).toBytes(),
DataSize.of(128, MEGABYTE)),
+ Pair.of(DataSize.of(384, MEGABYTE).toBytes(),
DataSize.of(116, MEGABYTE))));
+ }
+
+ @Test
+ public void testCreateHudiSplitsWithZeroTargetSplitSize()
+ {
+ // A zero target split size must be rejected on construction, before
any file slice is seen,
+ // instead of looping forever once split generation reaches a
non-empty base file
+ assertThatThrownBy(() -> new HudiSplitFactory(
+ createTableHandle(),
+ new SizeBasedSplitWeightProvider(0.05, DataSize.of(128,
MEGABYTE)),
+ DataSize.ofBytes(0)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("targetSplitSize");
}
@Test
@@ -151,8 +196,8 @@ public class TestHudiSplitFactory
FileSlice fileSlice = createFileSlice(baseFileSize, logFileSize);
- List<HudiSplit> splits = HudiSplitFactory.createHudiSplits(
- tableHandle, PARTITION_KEYS, fileSlice, COMMIT_TIME,
weightProvider, targetSplitSize);
+ List<HudiSplit> splits = new HudiSplitFactory(tableHandle,
weightProvider, targetSplitSize)
+ .createSplits(PARTITION_KEYS, fileSlice, COMMIT_TIME);
assertThat(splits).hasSize(expectedSplitInfo.size());
@@ -192,14 +237,17 @@ public class TestHudiSplitFactory
{
String fileId = "5a4f6a70-0306-40a8-952b-045b0d8ff0d4-0";
HoodieFileGroupId fileGroupId = new HoodieFileGroupId("partition",
fileId);
- long blockSize = 8L * 1024 * 1024;
+ // Block size mirrors the file length, which is what HudiTrinoStorage
now reports. Split
+ // generation must ignore it, so every multi-split expectation below
would collapse to a
+ // single whole-file split if the block size were allowed back into
the sizing decision.
String baseFilePath = "/test/path/" + fileGroupId + "_4-19-0_" +
COMMIT_TIME + ".parquet";
String logFilePath = "/test/path/." + fileId +
"_2025062515374131546.log.1_0-53-80";
+ long logFileSizeInBytes = logFileSize.isPresent() ?
logFileSize.get().toBytes() : 0L;
StoragePathInfo baseFileInfo = new StoragePathInfo(
- new StoragePath(baseFilePath), baseFileSize.toBytes(), false,
(short) 0, blockSize, System.currentTimeMillis());
+ new StoragePath(baseFilePath), baseFileSize.toBytes(), false,
(short) 0, baseFileSize.toBytes(), System.currentTimeMillis());
StoragePathInfo logFileInfo = new StoragePathInfo(
- new StoragePath(logFilePath), logFileSize.isPresent() ?
logFileSize.get().toBytes() : 0L,
- false, (short) 0, blockSize, System.currentTimeMillis());
+ new StoragePath(logFilePath), logFileSizeInBytes,
+ false, (short) 0, logFileSizeInBytes,
System.currentTimeMillis());
HoodieBaseFile baseFile = new HoodieBaseFile(baseFileInfo);
return new FileSlice(fileGroupId, COMMIT_TIME, baseFile,
logFileSize.isPresent() ? ImmutableList.of(new
HoodieLogFile(logFileInfo)) : ImmutableList.of());
diff --git
a/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java
b/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java
new file mode 100644
index 000000000000..a085a73af551
--- /dev/null
+++
b/hudi-trino/src/test/java/io/trino/plugin/hudi/storage/TestHudiTrinoStorage.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.storage;
+
+import io.trino.filesystem.FileEntry;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.memory.MemoryFileSystem;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class TestHudiTrinoStorage
+{
+ @Test
+ void testConvertToPathInfo()
+ {
+ FileEntry fileEntry = new FileEntry(
+ Location.of("memory:///table/data.parquet"),
+ 42,
+ Instant.ofEpochMilli(1234567890123L),
+ Optional.empty());
+
+ StoragePathInfo pathInfo =
HudiTrinoStorage.convertToPathInfo(fileEntry);
+
+ assertThat(pathInfo.getPath()).isEqualTo(new
StoragePath("memory:///table/data.parquet"));
+ assertThat(pathInfo.getLength()).isEqualTo(42);
+ assertThat(pathInfo.isFile()).isTrue();
+ assertThat(pathInfo.getBlockReplication()).isEqualTo((short) 0);
+ assertThat(pathInfo.getBlockSize()).isEqualTo(42);
+ assertThat(pathInfo.getModificationTime()).isEqualTo(1234567890123L);
+ }
+
+ @Test
+ void testGetPathInfoForFile()
+ throws IOException
+ {
+ TrinoFileSystem fileSystem = new MemoryFileSystem();
+ writeFile(fileSystem, "memory:///table/data.parquet", 42);
+ HudiTrinoStorage storage = new HudiTrinoStorage(fileSystem, new
TrinoStorageConfiguration());
+
+ StoragePathInfo pathInfo = storage.getPathInfo(new
StoragePath("memory:///table/data.parquet"));
+
+ assertThat(pathInfo.getLength()).isEqualTo(42);
+ assertThat(pathInfo.isFile()).isTrue();
+ assertThat(pathInfo.getBlockSize()).isEqualTo(42);
+ assertThat(pathInfo.getModificationTime()).isGreaterThan(0);
+ }
+
+ @Test
+ void testGetPathInfoForDirectory()
+ throws IOException
+ {
+ TrinoFileSystem fileSystem = new MemoryFileSystem();
+ writeFile(fileSystem, "memory:///table/data.parquet", 42);
+ HudiTrinoStorage storage = new HudiTrinoStorage(fileSystem, new
TrinoStorageConfiguration());
+
+ StoragePathInfo pathInfo = storage.getPathInfo(new
StoragePath("memory:///table"));
+
+ assertThat(pathInfo.isDirectory()).isTrue();
+ assertThat(pathInfo.getLength()).isEqualTo(0);
+ assertThat(pathInfo.getBlockSize()).isEqualTo(0);
+ }
+
+ @Test
+ void testListFiles()
+ throws IOException
+ {
+ HudiTrinoStorage storage = createStorageWithFiles();
+
+ List<StoragePathInfo> entries = storage.listFiles(new
StoragePath("memory:///table"));
+
+ assertThat(entries).hasSize(3);
+ assertThat(entries.get(0).getPath()).isEqualTo(new
StoragePath("memory:///table/a.parquet"));
+ assertThat(entries.get(1).getPath()).isEqualTo(new
StoragePath("memory:///table/b.parquet"));
+ assertThat(entries.get(2).getPath()).isEqualTo(new
StoragePath("memory:///table/nested/c.parquet"));
+ assertThat(entries.get(0).getLength()).isEqualTo(10);
+ assertThat(entries.get(1).getLength()).isEqualTo(20);
+ assertThat(entries.get(2).getLength()).isEqualTo(30);
+ for (StoragePathInfo entry : entries) {
+ assertThat(entry.getBlockSize()).isEqualTo(entry.getLength());
+ }
+ }
+
+ @Test
+ void testListDirectEntries()
+ throws IOException
+ {
+ HudiTrinoStorage storage = createStorageWithFiles();
+
+ List<StoragePathInfo> entries = storage.listDirectEntries(new
StoragePath("memory:///table"));
+
+ assertThat(entries).hasSize(2);
+ assertThat(entries.get(0).getPath()).isEqualTo(new
StoragePath("memory:///table/a.parquet"));
+ assertThat(entries.get(1).getPath()).isEqualTo(new
StoragePath("memory:///table/b.parquet"));
+ for (StoragePathInfo entry : entries) {
+ assertThat(entry.getBlockSize()).isEqualTo(entry.getLength());
+ }
+ }
+
+ @Test
+ void testListDirectEntriesWithFilter()
+ throws IOException
+ {
+ HudiTrinoStorage storage = createStorageWithFiles();
+
+ List<StoragePathInfo> entries = storage.listDirectEntries(
+ new StoragePath("memory:///table"),
+ path -> path.getName().equals("b.parquet"));
+
+ assertThat(entries).hasSize(1);
+ assertThat(entries.get(0).getPath()).isEqualTo(new
StoragePath("memory:///table/b.parquet"));
+ assertThat(entries.get(0).getLength()).isEqualTo(20);
+ assertThat(entries.get(0).getBlockSize()).isEqualTo(20);
+ }
+
+ private static HudiTrinoStorage createStorageWithFiles()
+ throws IOException
+ {
+ TrinoFileSystem fileSystem = new MemoryFileSystem();
+ writeFile(fileSystem, "memory:///table/a.parquet", 10);
+ writeFile(fileSystem, "memory:///table/b.parquet", 20);
+ writeFile(fileSystem, "memory:///table/nested/c.parquet", 30);
+ return new HudiTrinoStorage(fileSystem, new
TrinoStorageConfiguration());
+ }
+
+ private static void writeFile(TrinoFileSystem fileSystem, String location,
int length)
+ throws IOException
+ {
+ fileSystem.newOutputFile(Location.of(location)).createOrOverwrite(new
byte[length]);
+ }
+}
