(paimon) branch master updated: [core] Use rawConvertible to determine whether needs to convertToRawFiles (#3500)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 3353cfe9a [core] Use rawConvertible to determine whether needs to 
convertToRawFiles (#3500)
3353cfe9a is described below

commit 3353cfe9ac20dd0e699cbc79be6723a888e52388
Author: Zouxxyy <zouxinyu....@alibaba-inc.com>
AuthorDate: Tue Jun 11 15:38:32 2024 +0800

    [core] Use rawConvertible to determine whether needs to convertToRawFiles 
(#3500)
---
 .../org/apache/paimon/table/source/DataTableBatchScan.java    | 11 +++--------
 .../src/main/scala/org/apache/paimon/spark/ScanHelper.scala   |  2 +-
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git 
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
 
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
index 5c7b09fc2..8cf842aee 100644
--- 
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
+++ 
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
@@ -109,14 +109,9 @@ public class DataTableBatchScan extends 
AbstractDataTableScan {
      * merged.
      */
     private long getRowCountForSplit(DataSplit split) {
-        if (split.convertToRawFiles().isPresent()) {
-            return split.convertToRawFiles().get().stream()
-                    .map(RawFile::rowCount)
-                    .reduce(Long::sum)
-                    .orElse(0L);
-        } else {
-            return 0L;
-        }
+        return split.convertToRawFiles()
+                .map(files -> 
files.stream().map(RawFile::rowCount).reduce(Long::sum).orElse(0L))
+                .orElse(0L);
     }
 
     @Override
diff --git 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
index 96c1078ed..762ae0146 100644
--- 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
+++ 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
@@ -48,7 +48,7 @@ trait ScanHelper extends Logging {
     if (splits.length < leafNodeDefaultParallelism) {
       val beforeLength = splits.length
       val (toReshuffle, reserved) = splits.partition {
-        case split: DataSplit => split.beforeFiles().isEmpty && 
split.convertToRawFiles.isPresent
+        case split: DataSplit => split.beforeFiles().isEmpty && 
split.rawConvertible()
         case _ => false
       }
       val reshuffled = reshuffleSplits0(toReshuffle.collect { case ds: 
DataSplit => ds })