(incubator-paimon) branch master updated: [core] Extract file format of DataFileMeta through suffix of filename (#2389)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new d23607dfd [core] Extract file format of DataFileMeta through suffix of 
filename (#2389)
d23607dfd is described below

commit d23607dfd734c585770ef5d28b4c7a43e4c291fe
Author: tsreaper <tsreape...@gmail.com>
AuthorDate: Mon Nov 27 09:35:29 2023 +0800

    [core] Extract file format of DataFileMeta through suffix of filename 
(#2389)
---
 .../src/main/java/org/apache/paimon/io/DataFileMeta.java      | 11 +++++++++++
 .../paimon/table/source/snapshot/SnapshotReaderImpl.java      |  8 +++++++-
 .../paimon/table/source/snapshot/SnapshotReaderTest.java      |  7 ++++++-
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java 
b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
index ab20ff5ae..41bceea49 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
@@ -37,6 +37,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
 import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -223,6 +224,16 @@ public class DataFileMeta {
                 .toEpochMilli();
     }
 
+    public Optional<CoreOptions.FileFormatType> fileFormat() {
+        String[] split = fileName.split("\\.");
+        try {
+            return Optional.of(
+                    CoreOptions.FileFormatType.valueOf(split[split.length - 
1].toUpperCase()));
+        } catch (IllegalArgumentException e) {
+            return Optional.empty();
+        }
+    }
+
     public DataFileMeta upgrade(int newLevel) {
         checkArgument(newLevel > this.level);
         return new DataFileMeta(
diff --git 
a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
 
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
index 1c6bdc64d..d36eb76ce 100644
--- 
a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
+++ 
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
@@ -442,7 +442,13 @@ public class SnapshotReaderImpl implements SnapshotReader {
                 bucketPath + "/" + meta.fileName(),
                 0,
                 meta.fileSize(),
-                new 
CoreOptions(tableSchema.options()).formatType().toString().toLowerCase(),
+                meta.fileFormat()
+                        .map(t -> t.toString().toLowerCase())
+                        .orElse(
+                                new CoreOptions(tableSchema.options())
+                                        .formatType()
+                                        .toString()
+                                        .toLowerCase()),
                 meta.schemaId());
     }
 }
diff --git 
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
 
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
index edb3f0a86..a2b7faa9f 100644
--- 
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
+++ 
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
@@ -50,7 +50,9 @@ import org.junit.jupiter.api.io.TempDir;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -155,7 +157,7 @@ public class SnapshotReaderTest {
                                                     tablePath, partition, 
meta.fileName()),
                                             0,
                                             meta.fileSize(),
-                                            "avro",
+                                            meta.level() == 5 ? "orc" : "avro",
                                             meta.schemaId())));
         }
 
@@ -266,6 +268,9 @@ public class SnapshotReaderTest {
         options.set(CoreOptions.BUCKET, 1);
         options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 5);
         options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+        Map<String, String> formatPerLevel = new HashMap<>();
+        formatPerLevel.put("5", "orc");
+        options.set(CoreOptions.FILE_FORMAT_PER_LEVEL, formatPerLevel);
 
         SchemaManager schemaManager = new SchemaManager(fileIO, tablePath);
         TableSchema tableSchema =