This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-0.8 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0e56ff848ac5c5b12677762fa8602b9620c8f448 Author: Zouxxyy <[email protected]> AuthorDate: Tue Jun 11 15:38:32 2024 +0800 [core] Use rawConvertible to determine whether needs to convertToRawFiles (#3500) --- .../org/apache/paimon/table/source/InnerTableScanImpl.java | 11 +++-------- .../src/main/scala/org/apache/paimon/spark/ScanHelper.scala | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScanImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScanImpl.java index b307279d0..5474ab78f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScanImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScanImpl.java @@ -109,13 +109,8 @@ public class InnerTableScanImpl extends AbstractInnerTableScan { * merged. */ private long getRowCountForSplit(DataSplit split) { - if (split.convertToRawFiles().isPresent()) { - return split.convertToRawFiles().get().stream() - .map(RawFile::rowCount) - .reduce(Long::sum) - .orElse(0L); - } else { - return 0L; - } + return split.convertToRawFiles() + .map(files -> files.stream().map(RawFile::rowCount).reduce(Long::sum).orElse(0L)) + .orElse(0L); } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala index 8074322ea..11ac2ceaa 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala @@ -46,7 +46,7 @@ trait ScanHelper { def reshuffleSplits(splits: Array[Split]): Array[Split] = { if (splits.length < leafNodeDefaultParallelism) { val (toReshuffle, reserved) = splits.partition { - case split: DataSplit => split.beforeFiles().isEmpty && split.convertToRawFiles.isPresent + case split: DataSplit => split.beforeFiles().isEmpty && split.rawConvertible() case _ => false } val reshuffled = reshuffleSplits0(toReshuffle.collect { case ds: DataSplit => ds })
