This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6acc43459 [core] Optimize BinaryTableStats to real binary (#2800)
6acc43459 is described below
commit 6acc43459da5b3050a60b40edde6f7a1c0508c3d
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Jan 29 10:44:39 2024 +0800
[core] Optimize BinaryTableStats to real binary (#2800)
---
.../java/org/apache/paimon/data/BinaryArray.java | 33 +++
.../main/java/org/apache/paimon/predicate/And.java | 11 +-
.../apache/paimon/predicate/CompoundPredicate.java | 13 +-
.../java/org/apache/paimon/predicate/Equal.java | 7 +-
.../apache/paimon/predicate/GreaterOrEqual.java | 6 +-
.../org/apache/paimon/predicate/GreaterThan.java | 6 +-
.../main/java/org/apache/paimon/predicate/In.java | 13 +-
.../org/apache/paimon/predicate/IsNotNull.java | 4 +-
.../java/org/apache/paimon/predicate/IsNull.java | 4 +-
.../org/apache/paimon/predicate/LeafFunction.java | 8 +-
.../org/apache/paimon/predicate/LeafPredicate.java | 19 +-
.../apache/paimon/predicate/LeafUnaryFunction.java | 13 +-
.../org/apache/paimon/predicate/LessOrEqual.java | 6 +-
.../java/org/apache/paimon/predicate/LessThan.java | 6 +-
.../java/org/apache/paimon/predicate/NotEqual.java | 7 +-
.../java/org/apache/paimon/predicate/NotIn.java | 13 +-
.../predicate/NullFalseLeafBinaryFunction.java | 13 +-
.../main/java/org/apache/paimon/predicate/Or.java | 11 +-
.../org/apache/paimon/predicate/Predicate.java | 5 +-
.../org/apache/paimon/predicate/StartsWith.java | 16 +-
.../apache/paimon/predicate/FieldStatsUtils.java} | 41 ++--
.../paimon/predicate/PredicateBuilderTest.java | 26 ++-
.../org/apache/paimon/predicate/PredicateTest.java | 224 ++++++++++++---------
.../java/org/apache/paimon/io/DataFileMeta.java | 3 +-
.../apache/paimon/io/DataFileMetaSerializer.java | 8 +-
.../apache/paimon/manifest/ManifestFileMeta.java | 6 +-
.../manifest/ManifestFileMetaSerializer.java | 4 +-
.../paimon/operation/AbstractFileStoreScan.java | 13 +-
.../paimon/operation/AppendOnlyFileStoreScan.java | 22 +-
.../paimon/operation/KeyValueFileStoreScan.java | 35 ++--
.../paimon/partition/PartitionPredicate.java | 30 ++-
.../apache/paimon/schema/SchemaEvolutionUtil.java | 13 +-
.../org/apache/paimon/stats/BinaryTableStats.java | 103 +++-------
.../paimon/stats/FieldStatsArraySerializer.java | 214 +++++++++++++++++++-
.../apache/paimon/stats/FieldStatsConverters.java | 15 +-
.../org/apache/paimon/table/system/FilesTable.java | 2 +-
.../org/apache/paimon/utils/BulkFormatMapping.java | 2 +-
.../apache/paimon/append/AppendOnlyWriterTest.java | 13 +-
.../apache/paimon/manifest/ManifestFileTest.java | 10 +-
.../paimon/partition/PartitionPredicateTest.java | 16 +-
.../apache/paimon/stats/BinaryTableStatsTest.java | 93 ---------
.../stats/FieldStatsArraySerializerTest.java | 18 +-
.../org/apache/paimon/stats/StatsTableTest.java | 16 +-
.../paimon/table/ColumnTypeFileMetaTestBase.java | 145 ++++++-------
.../paimon/table/FileMetaFilterTestBase.java | 88 ++++----
.../PrimaryKeyTableColumnTypeFileMetaTest.java | 63 +++---
.../paimon/flink/PredicateConverterTest.java | 7 +-
47 files changed, 857 insertions(+), 587 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
index 981c8ecd6..0c3437fb4 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
@@ -567,4 +567,37 @@ public final class BinaryArray extends BinarySection
implements InternalArray, D
result.pointTo(MemorySegment.wrap(data), 0, (int) totalSize);
return result;
}
+
+ public static BinaryArray fromLongArray(Long[] arr) {
+ BinaryArray array = new BinaryArray();
+ BinaryArrayWriter writer = new BinaryArrayWriter(array, arr.length, 8);
+ for (int i = 0; i < arr.length; i++) {
+ Long v = arr[i];
+ if (v == null) {
+ writer.setNullLong(i);
+ } else {
+ writer.writeLong(i, v);
+ }
+ }
+ writer.complete();
+ return array;
+ }
+
+ public static BinaryArray fromLongArray(InternalArray arr) {
+ if (arr instanceof BinaryArray) {
+ return (BinaryArray) arr;
+ }
+
+ BinaryArray array = new BinaryArray();
+ BinaryArrayWriter writer = new BinaryArrayWriter(array, arr.size(), 8);
+ for (int i = 0; i < arr.size(); i++) {
+ if (arr.isNullAt(i)) {
+ writer.setNullLong(i);
+ } else {
+ writer.writeLong(i, arr.getLong(i));
+ }
+ }
+ writer.complete();
+ return array;
+ }
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/And.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/And.java
index c4ec83525..372e2c501 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/And.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/And.java
@@ -18,8 +18,8 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.format.FieldStats;
import java.util.ArrayList;
import java.util.List;
@@ -45,9 +45,14 @@ public class And extends CompoundPredicate.Function {
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats,
List<Predicate> children) {
+ public boolean test(
+ long rowCount,
+ InternalRow minValues,
+ InternalRow maxValues,
+ InternalArray nullCounts,
+ List<Predicate> children) {
for (Predicate child : children) {
- if (!child.test(rowCount, fieldStats)) {
+ if (!child.test(rowCount, minValues, maxValues, nullCounts)) {
return false;
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/CompoundPredicate.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/CompoundPredicate.java
index b8540722f..7b880c8a9 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/CompoundPredicate.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/CompoundPredicate.java
@@ -18,8 +18,8 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.format.FieldStats;
import java.io.Serializable;
import java.util.List;
@@ -54,8 +54,9 @@ public class CompoundPredicate implements Predicate {
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats) {
- return function.test(rowCount, fieldStats, children);
+ public boolean test(
+ long rowCount, InternalRow minValues, InternalRow maxValues,
InternalArray nullCounts) {
+ return function.test(rowCount, minValues, maxValues, nullCounts,
children);
}
@Override
@@ -93,7 +94,11 @@ public class CompoundPredicate implements Predicate {
public abstract boolean test(InternalRow row, List<Predicate>
children);
public abstract boolean test(
- long rowCount, FieldStats[] fieldStats, List<Predicate>
children);
+ long rowCount,
+ InternalRow minValues,
+ InternalRow maxValues,
+ InternalArray nullCounts,
+ List<Predicate> children);
public abstract Optional<Predicate> negate(List<Predicate> children);
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Equal.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/Equal.java
index e375479a6..e826fb0f1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Equal.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Equal.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,9 +38,9 @@ public class Equal extends NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.minValue()) >= 0
- && compareLiteral(type, literal, fieldStats.maxValue()) <= 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, min) >= 0 && compareLiteral(type,
literal, max) <= 0;
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterOrEqual.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterOrEqual.java
index 85005c55b..d4ec8639b 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterOrEqual.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterOrEqual.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,8 +38,9 @@ public class GreaterOrEqual extends
NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.maxValue()) <= 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, max) <= 0;
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterThan.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterThan.java
index 57300945c..0b498b3ab 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterThan.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/GreaterThan.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,8 +38,9 @@ public class GreaterThan extends NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.maxValue()) < 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, max) < 0;
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/In.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/In.java
index 2360b3f4f..ba96f6651 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/In.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/In.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -50,15 +49,19 @@ public class In extends LeafFunction {
@Override
public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals) {
- Long nullCount = fieldStats.nullCount();
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ List<Object> literals) {
if (nullCount != null && rowCount == nullCount) {
return false;
}
for (Object literal : literals) {
if (literal != null
- && compareLiteral(type, literal, fieldStats.minValue()) >= 0
- && compareLiteral(type, literal, fieldStats.maxValue()) <=
0) {
+ && compareLiteral(type, literal, min) >= 0
+ && compareLiteral(type, literal, max) <= 0) {
return true;
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/IsNotNull.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/IsNotNull.java
index dcf5d1687..3d2f9584e 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/IsNotNull.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/IsNotNull.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -37,8 +36,7 @@ public class IsNotNull extends LeafUnaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats) {
- Long nullCount = fieldStats.nullCount();
+ public boolean test(DataType type, long rowCount, Object min, Object max,
Long nullCount) {
return nullCount == null || nullCount < rowCount;
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/IsNull.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/IsNull.java
index 22300c718..f825b2e23 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/IsNull.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/IsNull.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -37,8 +36,7 @@ public class IsNull extends LeafUnaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats) {
- Long nullCount = fieldStats.nullCount();
+ public boolean test(DataType type, long rowCount, Object min, Object max,
Long nullCount) {
return nullCount == null || nullCount > 0;
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
index 075be7112..7a1e3d0f5 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.io.Serializable;
@@ -31,7 +30,12 @@ public abstract class LeafFunction implements Serializable {
public abstract boolean test(DataType type, Object field, List<Object>
literals);
public abstract boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals);
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ List<Object> literals);
public abstract Optional<LeafFunction> negate();
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
index d2a2461ee..5267f7069 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
@@ -18,15 +18,14 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.data.serializer.ListSerializer;
import org.apache.paimon.data.serializer.NullableSerializer;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.io.DataInputViewStreamWrapper;
import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.types.DataType;
-import org.apache.paimon.utils.InternalRowUtils;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -35,6 +34,8 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
+import static org.apache.paimon.utils.InternalRowUtils.get;
+
/** Leaf node of a {@link Predicate} tree. Compares a field in the row with
literals. */
public class LeafPredicate implements Predicate {
@@ -90,22 +91,24 @@ public class LeafPredicate implements Predicate {
@Override
public boolean test(InternalRow row) {
- return function.test(type, InternalRowUtils.get(row, fieldIndex,
type), literals);
+ return function.test(type, get(row, fieldIndex, type), literals);
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats) {
- FieldStats stats = fieldStats[fieldIndex];
- Long nullCount = stats.nullCount();
+ public boolean test(
+ long rowCount, InternalRow minValues, InternalRow maxValues,
InternalArray nullCounts) {
+ Object min = get(minValues, fieldIndex, type);
+ Object max = get(maxValues, fieldIndex, type);
+ Long nullCount = nullCounts.isNullAt(fieldIndex) ? null :
nullCounts.getLong(fieldIndex);
if (nullCount == null || rowCount != nullCount) {
// not all null
// min or max is null
// unknown stats
- if (stats.minValue() == null || stats.maxValue() == null) {
+ if (min == null || max == null) {
return true;
}
}
- return function.test(type, rowCount, stats, literals);
+ return function.test(type, rowCount, min, max, nullCount, literals);
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
index 429411c24..575ef8371 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -30,7 +29,8 @@ public abstract class LeafUnaryFunction extends LeafFunction {
public abstract boolean test(DataType type, Object value);
- public abstract boolean test(DataType type, long rowCount, FieldStats
fieldStats);
+ public abstract boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount);
@Override
public boolean test(DataType type, Object value, List<Object> literals) {
@@ -39,7 +39,12 @@ public abstract class LeafUnaryFunction extends LeafFunction
{
@Override
public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals) {
- return test(type, rowCount, fieldStats);
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ List<Object> literals) {
+ return test(type, rowCount, min, max, nullCount);
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LessOrEqual.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/LessOrEqual.java
index 42bfe1192..cc17decad 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LessOrEqual.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LessOrEqual.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,8 +38,9 @@ public class LessOrEqual extends NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.minValue()) >= 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, min) >= 0;
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LessThan.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/LessThan.java
index 8149f550c..e26c8d68c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LessThan.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LessThan.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,8 +38,9 @@ public class LessThan extends NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.minValue()) > 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, min) > 0;
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/NotEqual.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/NotEqual.java
index 0e2704ada..a3bdfbb99 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/NotEqual.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotEqual.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -39,9 +38,9 @@ public class NotEqual extends NullFalseLeafBinaryFunction {
}
@Override
- public boolean test(DataType type, long rowCount, FieldStats fieldStats,
Object literal) {
- return compareLiteral(type, literal, fieldStats.minValue()) != 0
- || compareLiteral(type, literal, fieldStats.maxValue()) != 0;
+ public boolean test(
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal) {
+ return compareLiteral(type, literal, min) != 0 || compareLiteral(type,
literal, max) != 0;
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/NotIn.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/NotIn.java
index 0a0a014df..d3a9715bf 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/NotIn.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotIn.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -50,15 +49,19 @@ public class NotIn extends LeafFunction {
@Override
public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals) {
- Long nullCount = fieldStats.nullCount();
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ List<Object> literals) {
if (nullCount != null && rowCount == nullCount) {
return false;
}
for (Object literal : literals) {
if (literal == null
- || (compareLiteral(type, literal, fieldStats.minValue())
== 0
- && compareLiteral(type, literal,
fieldStats.maxValue()) == 0)) {
+ || (compareLiteral(type, literal, min) == 0
+ && compareLiteral(type, literal, max) == 0)) {
return false;
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/NullFalseLeafBinaryFunction.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/NullFalseLeafBinaryFunction.java
index 244ed3d72..2c8b57d0d 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/NullFalseLeafBinaryFunction.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/NullFalseLeafBinaryFunction.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -31,7 +30,7 @@ public abstract class NullFalseLeafBinaryFunction extends
LeafFunction {
public abstract boolean test(DataType type, Object field, Object literal);
public abstract boolean test(
- DataType type, long rowCount, FieldStats fieldStats, Object
literal);
+ DataType type, long rowCount, Object min, Object max, Long
nullCount, Object literal);
@Override
public boolean test(DataType type, Object field, List<Object> literals) {
@@ -43,13 +42,17 @@ public abstract class NullFalseLeafBinaryFunction extends
LeafFunction {
@Override
public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals) {
- Long nullCount = fieldStats.nullCount();
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ List<Object> literals) {
if (nullCount != null) {
if (rowCount == nullCount || literals.get(0) == null) {
return false;
}
}
- return test(type, rowCount, fieldStats, literals.get(0));
+ return test(type, rowCount, min, max, nullCount, literals.get(0));
}
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Or.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/Or.java
index c145878a2..3385677ab 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Or.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Or.java
@@ -18,8 +18,8 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.format.FieldStats;
import java.util.ArrayList;
import java.util.List;
@@ -45,9 +45,14 @@ public class Or extends CompoundPredicate.Function {
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats,
List<Predicate> children) {
+ public boolean test(
+ long rowCount,
+ InternalRow minValues,
+ InternalRow maxValues,
+ InternalArray nullCounts,
+ List<Predicate> children) {
for (Predicate child : children) {
- if (child.test(rowCount, fieldStats)) {
+ if (child.test(rowCount, minValues, maxValues, nullCounts)) {
return true;
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/Predicate.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/Predicate.java
index c3d0cd8eb..b04029ba7 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Predicate.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Predicate.java
@@ -19,8 +19,8 @@
package org.apache.paimon.predicate;
import org.apache.paimon.annotation.Public;
+import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.format.FieldStats;
import java.io.Serializable;
import java.util.Optional;
@@ -47,7 +47,8 @@ public interface Predicate extends Serializable {
* @return return true is likely to hit (there may also be false
positives), return false is
* absolutely not possible to hit.
*/
- boolean test(long rowCount, FieldStats[] fieldStats);
+ boolean test(
+ long rowCount, InternalRow minValues, InternalRow maxValues,
InternalArray nullCounts);
/** @return the negation predicate of this predicate if possible. */
Optional<Predicate> negate();
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
index acbdd6de1..56e5e8ecf 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
@@ -19,7 +19,6 @@
package org.apache.paimon.predicate;
import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.DataType;
import java.util.List;
@@ -43,12 +42,17 @@ public class StartsWith extends NullFalseLeafBinaryFunction
{
@Override
public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, Object
patternLiteral) {
- BinaryString min = (BinaryString) fieldStats.minValue();
- BinaryString max = (BinaryString) fieldStats.maxValue();
+ DataType type,
+ long rowCount,
+ Object min,
+ Object max,
+ Long nullCount,
+ Object patternLiteral) {
+ BinaryString minStr = (BinaryString) min;
+ BinaryString maxStr = (BinaryString) max;
BinaryString pattern = (BinaryString) patternLiteral;
- return (min.startsWith(pattern) || min.compareTo(pattern) <= 0)
- && (max.startsWith(pattern) || max.compareTo(pattern) >= 0);
+ return (minStr.startsWith(pattern) || minStr.compareTo(pattern) <= 0)
+ && (maxStr.startsWith(pattern) || maxStr.compareTo(pattern) >=
0);
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/FieldStatsUtils.java
similarity index 52%
copy from
paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
copy to
paimon-common/src/test/java/org/apache/paimon/predicate/FieldStatsUtils.java
index 429411c24..b829cf34d 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafUnaryFunction.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/FieldStatsUtils.java
@@ -18,28 +18,27 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.data.BinaryArray;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.format.FieldStats;
-import org.apache.paimon.types.DataType;
-import java.util.List;
-
-/** Function to test a field. */
-public abstract class LeafUnaryFunction extends LeafFunction {
-
- private static final long serialVersionUID = 1L;
-
- public abstract boolean test(DataType type, Object value);
-
- public abstract boolean test(DataType type, long rowCount, FieldStats
fieldStats);
-
- @Override
- public boolean test(DataType type, Object value, List<Object> literals) {
- return test(type, value);
- }
-
- @Override
- public boolean test(
- DataType type, long rowCount, FieldStats fieldStats, List<Object>
literals) {
- return test(type, rowCount, fieldStats);
+/** Utils for testing with {@link FieldStats}. */
+public class FieldStatsUtils {
+
+ public static boolean test(Predicate predicate, long rowCount,
FieldStats[] fieldStats) {
+ Object[] min = new Object[fieldStats.length];
+ Object[] max = new Object[fieldStats.length];
+ Long[] nullCounts = new Long[fieldStats.length];
+ for (int i = 0; i < fieldStats.length; i++) {
+ min[i] = fieldStats[i].minValue();
+ max[i] = fieldStats[i].maxValue();
+ nullCounts[i] = fieldStats[i].nullCount();
+ }
+
+ return predicate.test(
+ rowCount,
+ GenericRow.of(min),
+ GenericRow.of(max),
+ BinaryArray.fromLongArray(nullCounts));
}
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
index b88240333..8990c2427 100644
---
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test;
import java.util.Arrays;
+import static org.apache.paimon.predicate.FieldStatsUtils.test;
import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link PredicateBuilder}. */
@@ -43,11 +44,12 @@ public class PredicateBuilderTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(2, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 2,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(2, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 2,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -62,11 +64,15 @@ public class PredicateBuilderTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(2, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 2,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(2, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 2,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
index ceac81b38..d449f55e4 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import static org.apache.paimon.predicate.FieldStatsUtils.test;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link Predicate}s. */
@@ -43,10 +44,11 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(5))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.notEqual(0, 5));
@@ -60,8 +62,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -74,11 +77,12 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(5))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(5, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(5, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.equal(0,
5));
@@ -92,8 +96,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -107,11 +112,13 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(6))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 4,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 4,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.lessOrEqual(0,
5));
@@ -125,8 +132,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 4,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 4,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -140,11 +148,12 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(6))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 4,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 4,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 6,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.lessThan(0, 5));
@@ -158,8 +167,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 4,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 4,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -173,10 +183,12 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(6))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(5, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(4, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(5, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(4, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.greaterOrEqual(0,
5));
@@ -190,8 +202,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -205,10 +218,11 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(6))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(5, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(4, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(5, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(4, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.greaterThan(0,
5));
@@ -222,8 +236,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -235,8 +250,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(5, 7,
1L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(5, 7,
1L)})).isEqualTo(true);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.isNotNull(0));
}
@@ -249,9 +265,9 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(4))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(5, 7,
1L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(null,
null, 3L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(5, 7,
1L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(null,
null, 3L)}))
.isEqualTo(false);
assertThat(predicate.negate().orElse(null)).isEqualTo(builder.isNull(0));
@@ -268,9 +284,10 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -285,9 +302,10 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -302,12 +320,14 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 1,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(3, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 1,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(3, 3,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -322,12 +342,17 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 1,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(3, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 1,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(3, 3,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 3,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
}
@@ -348,11 +373,12 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
.isEqualTo(true);
}
@@ -374,11 +400,12 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(true);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
.isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
.isEqualTo(true);
}
@@ -399,14 +426,16 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 1,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(3, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 1,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(3, 3,
0L)}))
.isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(true);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
.isEqualTo(true);
}
@@ -428,14 +457,19 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(3))).isEqualTo(false);
assertThat(predicate.test(GenericRow.of((Object)
null))).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 1,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(3, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(1, 3,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(0, 5,
0L)})).isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(6, 7,
0L)})).isEqualTo(false);
- assertThat(predicate.test(1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 1,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(3, 3,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(1, 3,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(0, 5,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(6, 7,
0L)}))
+ .isEqualTo(false);
+ assertThat(test(predicate, 1, new FieldStats[] {new FieldStats(null,
null, 1L)}))
.isEqualTo(false);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(29, 32,
0L)}))
.isEqualTo(false);
}
@@ -450,21 +484,24 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(null, 5))).isEqualTo(false);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(3, 6, 0L), new
FieldStats(4, 6, 0L)
}))
.isEqualTo(true);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(3, 6, 0L), new
FieldStats(6, 8, 0L)
}))
.isEqualTo(false);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(6, 7, 0L), new
FieldStats(4, 6, 0L)
@@ -486,21 +523,24 @@ public class PredicateTest {
assertThat(predicate.test(GenericRow.of(null, 5))).isEqualTo(true);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(3, 6, 0L), new
FieldStats(4, 6, 0L)
}))
.isEqualTo(true);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(3, 6, 0L), new
FieldStats(6, 8, 0L)
}))
.isEqualTo(true);
assertThat(
- predicate.test(
+ test(
+ predicate,
3,
new FieldStats[] {
new FieldStats(6, 7, 0L), new
FieldStats(8, 10, 0L)
@@ -516,11 +556,11 @@ public class PredicateTest {
PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
Predicate predicate = builder.equal(0, 5);
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(null,
null, 3L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(null,
null, 3L)}))
.isEqualTo(false);
// unknown stats, we don't know, likely to hit
- assertThat(predicate.test(3, new FieldStats[] {new FieldStats(null,
null, 4L)}))
+ assertThat(test(predicate, 3, new FieldStats[] {new FieldStats(null,
null, 4L)}))
.isEqualTo(true);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
index 41bceea49..0b1c5c50d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
@@ -19,6 +19,7 @@
package org.apache.paimon.io;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.fs.Path;
@@ -50,7 +51,7 @@ public class DataFileMeta {
// Append only data files don't have any key columns and meaningful level
value. it will use
// the following dummy values.
public static final BinaryTableStats EMPTY_KEY_STATS =
- new BinaryTableStats(EMPTY_ROW, EMPTY_ROW, new Long[0]);
+ new BinaryTableStats(EMPTY_ROW, EMPTY_ROW,
BinaryArray.fromLongArray(new Long[0]));
public static final BinaryRow EMPTY_MIN_KEY = EMPTY_ROW;
public static final BinaryRow EMPTY_MAX_KEY = EMPTY_ROW;
public static final int DUMMY_LEVEL = 0;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java
index dad2bc973..a823b5791 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java
@@ -46,8 +46,8 @@ public class DataFileMetaSerializer extends
ObjectSerializer<DataFileMeta> {
meta.rowCount(),
serializeBinaryRow(meta.minKey()),
serializeBinaryRow(meta.maxKey()),
- meta.keyStats().toRowData(),
- meta.valueStats().toRowData(),
+ meta.keyStats().toRow(),
+ meta.valueStats().toRow(),
meta.minSequenceNumber(),
meta.maxSequenceNumber(),
meta.schemaId(),
@@ -64,8 +64,8 @@ public class DataFileMetaSerializer extends
ObjectSerializer<DataFileMeta> {
row.getLong(2),
deserializeBinaryRow(row.getBinary(3)),
deserializeBinaryRow(row.getBinary(4)),
- BinaryTableStats.fromRowData(row.getRow(5, 3)),
- BinaryTableStats.fromRowData(row.getRow(6, 3)),
+ BinaryTableStats.fromRow(row.getRow(5, 3)),
+ BinaryTableStats.fromRow(row.getRow(6, 3)),
row.getLong(7),
row.getLong(8),
row.getLong(9),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java
index b917e27fb..5910ad75f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java
@@ -281,14 +281,14 @@ public class ManifestFileMeta {
if (predicateOpt.isPresent()) {
Predicate predicate = predicateOpt.get();
- FieldStatsArraySerializer fieldStatsArraySerializer =
- new FieldStatsArraySerializer(partitionType);
for (; j < base.size(); j++) {
// TODO: optimize this to binary search.
ManifestFileMeta file = base.get(j);
if (predicate.test(
file.numAddedFiles + file.numDeletedFiles,
-
file.partitionStats.fields(fieldStatsArraySerializer))) {
+ file.partitionStats.minValues(),
+ file.partitionStats.maxValues(),
+ file.partitionStats.nullCounts())) {
break;
} else {
result.add(file);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
index bba5803a1..632644f85 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
@@ -45,7 +45,7 @@ public class ManifestFileMetaSerializer extends
VersionedObjectSerializer<Manife
meta.fileSize(),
meta.numAddedFiles(),
meta.numDeletedFiles(),
- meta.partitionStats().toRowData(),
+ meta.partitionStats().toRow(),
meta.schemaId());
}
@@ -65,7 +65,7 @@ public class ManifestFileMetaSerializer extends
VersionedObjectSerializer<Manife
row.getLong(1),
row.getLong(2),
row.getLong(3),
- BinaryTableStats.fromRowData(row.getRow(4, 3)),
+ BinaryTableStats.fromRow(row.getRow(4, 3)),
row.getLong(5));
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
index 94a383f95..65857daed 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
@@ -33,7 +33,7 @@ import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
-import org.apache.paimon.stats.FieldStatsArraySerializer;
+import org.apache.paimon.stats.BinaryTableStats;
import org.apache.paimon.table.source.ScanMode;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FileStorePathFactory;
@@ -61,7 +61,6 @@ import static
org.apache.paimon.utils.Preconditions.checkState;
/** Default implementation of {@link FileStoreScan}. */
public abstract class AbstractFileStoreScan implements FileStoreScan {
- private final FieldStatsArraySerializer partitionStatsConverter;
private final RowType partitionType;
private final SnapshotManager snapshotManager;
private final ManifestFile.Factory manifestFileFactory;
@@ -96,7 +95,6 @@ public abstract class AbstractFileStoreScan implements
FileStoreScan {
int numOfBuckets,
boolean checkNumOfBuckets,
Integer scanManifestParallelism) {
- this.partitionStatsConverter = new
FieldStatsArraySerializer(partitionType);
this.partitionType = partitionType;
this.bucketKeyFilter = bucketKeyFilter;
this.snapshotManager = snapshotManager;
@@ -379,10 +377,17 @@ public abstract class AbstractFileStoreScan implements
FileStoreScan {
/** Note: Keep this thread-safe. */
private boolean filterManifestFileMeta(ManifestFileMeta manifest) {
+ if (partitionFilter == null) {
+ return true;
+ }
+
+ BinaryTableStats stats = manifest.partitionStats();
return partitionFilter == null
|| partitionFilter.test(
manifest.numAddedFiles() + manifest.numDeletedFiles(),
-
manifest.partitionStats().fields(partitionStatsConverter));
+ stats.minValues(),
+ stats.maxValues(),
+ stats.nullCounts());
}
/** Note: Keep this thread-safe. */
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
index c53966f0f..1cd7db0d2 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
@@ -24,6 +24,8 @@ import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestList;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.stats.BinaryTableStats;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
import org.apache.paimon.stats.FieldStatsConverters;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.SnapshotManager;
@@ -71,14 +73,18 @@ public class AppendOnlyFileStoreScan extends
AbstractFileStoreScan {
/** Note: Keep this thread-safe. */
@Override
protected boolean filterByStats(ManifestEntry entry) {
- return filter == null
- || filter.test(
- entry.file().rowCount(),
- entry.file()
- .valueStats()
- .fields(
-
fieldStatsConverters.getOrCreate(entry.file().schemaId()),
- entry.file().rowCount()));
+ if (filter == null) {
+ return true;
+ }
+
+ FieldStatsArraySerializer serializer =
+ fieldStatsConverters.getOrCreate(entry.file().schemaId());
+ BinaryTableStats stats = entry.file().valueStats();
+ return filter.test(
+ entry.file().rowCount(),
+ serializer.evolution(stats.minValues()),
+ serializer.evolution(stats.maxValues()),
+ serializer.evolution(stats.nullCounts(),
entry.file().rowCount()));
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
index 0c9d4a872..d528e5e79 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
@@ -25,6 +25,8 @@ import org.apache.paimon.manifest.ManifestList;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.schema.KeyValueFieldsExtractor;
import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.stats.BinaryTableStats;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
import org.apache.paimon.stats.FieldStatsConverters;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.SnapshotManager;
@@ -84,15 +86,18 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
/** Note: Keep this thread-safe. */
@Override
protected boolean filterByStats(ManifestEntry entry) {
- return keyFilter == null
- || keyFilter.test(
- entry.file().rowCount(),
- entry.file()
- .keyStats()
- .fields(
- fieldKeyStatsConverters.getOrCreate(
- entry.file().schemaId()),
- entry.file().rowCount()));
+ if (keyFilter == null) {
+ return true;
+ }
+
+ FieldStatsArraySerializer serializer =
+ fieldKeyStatsConverters.getOrCreate(entry.file().schemaId());
+ BinaryTableStats stats = entry.file().keyStats();
+ return keyFilter.test(
+ entry.file().rowCount(),
+ serializer.evolution(stats.minValues()),
+ serializer.evolution(stats.maxValues()),
+ serializer.evolution(stats.nullCounts(),
entry.file().rowCount()));
}
/** Note: Keep this thread-safe. */
@@ -102,14 +107,14 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
// the bucket.
if (valueFilter != null) {
for (ManifestEntry entry : entries) {
+ FieldStatsArraySerializer serializer =
+
fieldValueStatsConverters.getOrCreate(entry.file().schemaId());
+ BinaryTableStats stats = entry.file().valueStats();
if (valueFilter.test(
entry.file().rowCount(),
- entry.file()
- .valueStats()
- .fields(
- fieldValueStatsConverters.getOrCreate(
- entry.file().schemaId()),
- entry.file().rowCount()))) {
+ serializer.evolution(stats.minValues()),
+ serializer.evolution(stats.maxValues()),
+ serializer.evolution(stats.nullCounts(),
entry.file().rowCount()))) {
return true;
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java
index 2748ef40b..b0a89e29e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java
@@ -19,6 +19,8 @@
package org.apache.paimon.partition;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.data.serializer.Serializer;
import org.apache.paimon.format.FieldStats;
@@ -37,7 +39,8 @@ public interface PartitionPredicate {
boolean test(BinaryRow part);
- boolean test(long rowCount, FieldStats[] fieldStats);
+ boolean test(
+ long rowCount, InternalRow minValues, InternalRow maxValues,
InternalArray nullCounts);
static PartitionPredicate fromPredicate(Predicate predicate) {
return new DefaultPartitionPredicate(predicate);
@@ -63,8 +66,12 @@ public interface PartitionPredicate {
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats) {
- return predicate.test(rowCount, fieldStats);
+ public boolean test(
+ long rowCount,
+ InternalRow minValues,
+ InternalRow maxValues,
+ InternalArray nullCounts) {
+ return predicate.test(rowCount, minValues, maxValues, nullCounts);
}
}
@@ -75,7 +82,7 @@ public interface PartitionPredicate {
class MultiplePartitionPredicate implements PartitionPredicate {
private final Set<BinaryRow> partitions;
-
+ private final int fieldNum;
private final Predicate[] min;
private final Predicate[] max;
@@ -83,7 +90,7 @@ public interface PartitionPredicate {
RowDataToObjectArrayConverter converter, Set<BinaryRow>
partitions) {
this.partitions = partitions;
RowType partitionType = converter.rowType();
- int fieldNum = partitionType.getFieldCount();
+ this.fieldNum = partitionType.getFieldCount();
@SuppressWarnings("unchecked")
Serializer<Object>[] serializers = new Serializer[fieldNum];
FullFieldStatsCollector[] collectors = new
FullFieldStatsCollector[fieldNum];
@@ -113,13 +120,18 @@ public interface PartitionPredicate {
}
@Override
- public boolean test(long rowCount, FieldStats[] fieldStats) {
- if (fieldStats.length == 0) {
+ public boolean test(
+ long rowCount,
+ InternalRow minValues,
+ InternalRow maxValues,
+ InternalArray nullCounts) {
+ if (fieldNum == 0) {
return true;
}
- for (int i = 0; i < fieldStats.length; i++) {
- if (!min[i].test(rowCount, fieldStats) ||
!max[i].test(rowCount, fieldStats)) {
+ for (int i = 0; i < fieldNum; i++) {
+ if (!min[i].test(rowCount, minValues, maxValues, nullCounts)
+ || !max[i].test(rowCount, minValues, maxValues,
nullCounts)) {
return false;
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java
index ad119f772..8fa41bebf 100644
---
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java
+++
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java
@@ -139,12 +139,17 @@ public class SchemaEvolutionUtil {
List<DataField> tableFields,
int[] dataProjection,
List<DataField> dataFields) {
- List<DataField> tableProjectFields =
projectDataFields(tableProjection, tableFields);
- List<DataField> dataProjectFields = projectDataFields(dataProjection,
dataFields);
+ return createIndexCastMapping(
+ projectDataFields(tableProjection, tableFields),
+ projectDataFields(dataProjection, dataFields));
+ }
- int[] indexMapping = createIndexMapping(tableProjectFields,
dataProjectFields);
+ /** Create index mapping from table fields to underlying data fields. */
+ public static IndexCastMapping createIndexCastMapping(
+ List<DataField> tableFields, List<DataField> dataFields) {
+ int[] indexMapping = createIndexMapping(tableFields, dataFields);
CastFieldGetter[] castMapping =
- createCastFieldGetterMapping(tableProjectFields,
dataProjectFields, indexMapping);
+ createCastFieldGetterMapping(tableFields, dataFields,
indexMapping);
return new IndexCastMapping() {
@Nullable
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/stats/BinaryTableStats.java
b/paimon-core/src/main/java/org/apache/paimon/stats/BinaryTableStats.java
index 7923b0a4b..89bf9704b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/stats/BinaryTableStats.java
+++ b/paimon-core/src/main/java/org/apache/paimon/stats/BinaryTableStats.java
@@ -18,101 +18,52 @@
package org.apache.paimon.stats;
+import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FieldStats;
-import javax.annotation.Nullable;
-
-import java.util.Arrays;
import java.util.Objects;
-import static org.apache.paimon.utils.Preconditions.checkNotNull;
import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
/** A serialized row bytes to cache {@link FieldStats}. */
public class BinaryTableStats {
- @Nullable private InternalRow row;
- @Nullable private FieldStats[] cacheArray;
- @Nullable private BinaryRow cacheMin;
- @Nullable private BinaryRow cacheMax;
- @Nullable private Long[] cacheNullCounts;
-
- public BinaryTableStats(InternalRow row) {
- this.row = row;
- }
+ private final BinaryRow minValues;
+ private final BinaryRow maxValues;
+ private final BinaryArray nullCounts;
- public BinaryTableStats(BinaryRow cacheMin, BinaryRow cacheMax, Long[]
cacheNullCounts) {
- this(cacheMin, cacheMax, cacheNullCounts, null);
+ public BinaryTableStats(BinaryRow minValues, BinaryRow maxValues,
BinaryArray nullCounts) {
+ this.minValues = minValues;
+ this.maxValues = maxValues;
+ this.nullCounts = nullCounts;
}
- public BinaryTableStats(
- BinaryRow cacheMin,
- BinaryRow cacheMax,
- Long[] cacheNullCounts,
- @Nullable FieldStats[] cacheArray) {
- this.cacheMin = cacheMin;
- this.cacheMax = cacheMax;
- this.cacheNullCounts = cacheNullCounts;
- this.cacheArray = cacheArray;
+ public BinaryRow minValues() {
+ return minValues;
}
- public FieldStats[] fields(FieldStatsArraySerializer converter) {
- return fields(converter, null);
+ public BinaryRow maxValues() {
+ return maxValues;
}
- public FieldStats[] fields(FieldStatsArraySerializer converter, @Nullable
Long rowCount) {
- if (cacheArray == null) {
- cacheArray = converter.fromBinary(this, rowCount);
- }
- return cacheArray;
- }
-
- public BinaryRow min() {
- if (cacheMin == null) {
- checkNotNull(row);
- cacheMin = deserializeBinaryRow(this.row.getBinary(0));
- }
- return cacheMin;
- }
-
- public BinaryRow max() {
- if (cacheMax == null) {
- checkNotNull(row);
- cacheMax = deserializeBinaryRow(this.row.getBinary(1));
- }
- return cacheMax;
- }
-
- public Long[] nullCounts() {
- if (cacheNullCounts == null) {
- checkNotNull(row);
- InternalArray internalArray = row.getArray(2);
- Long[] array = new Long[internalArray.size()];
- for (int i = 0; i < array.length; i++) {
- array[i] = internalArray.isNullAt(i) ? null :
internalArray.getLong(i);
- }
- return array;
- }
- return cacheNullCounts;
+ public BinaryArray nullCounts() {
+ return nullCounts;
}
- public InternalRow toRowData() {
- return row == null
- ? GenericRow.of(
- serializeBinaryRow(min()),
- serializeBinaryRow(max()),
- new GenericArray(nullCounts()))
- : row;
+ public InternalRow toRow() {
+ return GenericRow.of(
+ serializeBinaryRow(minValues), serializeBinaryRow(maxValues),
nullCounts);
}
- public static BinaryTableStats fromRowData(InternalRow row) {
- return new BinaryTableStats(row);
+ public static BinaryTableStats fromRow(InternalRow row) {
+ return new BinaryTableStats(
+ deserializeBinaryRow(row.getBinary(0)),
+ deserializeBinaryRow(row.getBinary(1)),
+ BinaryArray.fromLongArray(row.getArray(2)));
}
@Override
@@ -124,15 +75,13 @@ public class BinaryTableStats {
return false;
}
BinaryTableStats that = (BinaryTableStats) o;
- return Objects.equals(min(), that.min())
- && Objects.equals(max(), that.max())
- && Arrays.equals(nullCounts(), that.nullCounts());
+ return Objects.equals(minValues, that.minValues)
+ && Objects.equals(maxValues, that.maxValues)
+ && Objects.equals(nullCounts, that.nullCounts);
}
@Override
public int hashCode() {
- int result = Objects.hash(min(), max());
- result = 31 * result + Arrays.hashCode(nullCounts());
- return result;
+ return Objects.hash(minValues, maxValues, nullCounts);
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
index 17db3a53e..0d830f551 100644
---
a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsArraySerializer.java
@@ -19,8 +19,17 @@
package org.apache.paimon.stats;
import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.casting.CastFieldGetter;
+import org.apache.paimon.casting.CastedRow;
+import org.apache.paimon.data.BinaryArray;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.format.FieldStats;
import org.apache.paimon.types.ArrayType;
@@ -29,6 +38,7 @@ import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.InternalRowUtils;
+import org.apache.paimon.utils.ProjectedRow;
import javax.annotation.Nullable;
@@ -47,13 +57,17 @@ public class FieldStatsArraySerializer {
@Nullable private final int[] indexMapping;
@Nullable private final CastExecutor<Object, Object>[] converterMapping;
+ @Nullable private final CastFieldGetter[] castFieldGetters;
public FieldStatsArraySerializer(RowType type) {
- this(type, null, null);
+ this(type, null, null, null);
}
public FieldStatsArraySerializer(
- RowType type, int[] indexMapping, CastExecutor<Object, Object>[]
converterMapping) {
+ RowType type,
+ @Nullable int[] indexMapping,
+ @Nullable CastExecutor<Object, Object>[] converterMapping,
+ @Nullable CastFieldGetter[] castFieldGetters) {
RowType safeType = toAllFieldsNullableRowType(type);
this.serializer = new InternalRowSerializer(safeType);
this.fieldGetters =
@@ -65,6 +79,7 @@ public class FieldStatsArraySerializer {
.toArray(InternalRow.FieldGetter[]::new);
this.indexMapping = indexMapping;
this.converterMapping = converterMapping;
+ this.castFieldGetters = castFieldGetters;
}
public BinaryTableStats toBinary(FieldStats[] stats) {
@@ -80,8 +95,7 @@ public class FieldStatsArraySerializer {
return new BinaryTableStats(
serializer.toBinaryRow(minValues).copy(),
serializer.toBinaryRow(maxValues).copy(),
- nullCounts,
- stats);
+ BinaryArray.fromLongArray(nullCounts));
}
public FieldStats[] fromBinary(BinaryTableStats array) {
@@ -91,10 +105,10 @@ public class FieldStatsArraySerializer {
public FieldStats[] fromBinary(BinaryTableStats array, @Nullable Long
rowCount) {
int fieldCount = indexMapping == null ? fieldGetters.length :
indexMapping.length;
FieldStats[] stats = new FieldStats[fieldCount];
- Long[] nullCounts = array.nullCounts();
+ BinaryArray nullCounts = array.nullCounts();
for (int i = 0; i < fieldCount; i++) {
int fieldIndex = indexMapping == null ? i : indexMapping[i];
- if (fieldIndex < 0 || fieldIndex >= array.min().getFieldCount()) {
+ if (fieldIndex < 0 || fieldIndex >=
array.minValues().getFieldCount()) {
// simple evolution for add column
if (rowCount == null) {
throw new RuntimeException("Schema Evolution for stats
needs row count.");
@@ -103,18 +117,49 @@ public class FieldStatsArraySerializer {
} else {
CastExecutor<Object, Object> converter =
converterMapping == null ? null : converterMapping[i];
- Object min =
fieldGetters[fieldIndex].getFieldOrNull(array.min());
+ Object min =
fieldGetters[fieldIndex].getFieldOrNull(array.minValues());
min = converter == null || min == null ? min :
converter.cast(min);
- Object max =
fieldGetters[fieldIndex].getFieldOrNull(array.max());
+ Object max =
fieldGetters[fieldIndex].getFieldOrNull(array.maxValues());
max = converter == null || max == null ? max :
converter.cast(max);
- stats[i] = new FieldStats(min, max, nullCounts[fieldIndex]);
+ stats[i] =
+ new FieldStats(
+ min,
+ max,
+ nullCounts.isNullAt(fieldIndex)
+ ? null
+ : nullCounts.getLong(fieldIndex));
}
}
return stats;
}
+ public InternalRow evolution(BinaryRow values) {
+ InternalRow row = values;
+ if (indexMapping != null) {
+ row = ProjectedRow.from(indexMapping).replaceRow(row);
+ }
+
+ if (castFieldGetters != null) {
+ row = CastedRow.from(castFieldGetters).replaceRow(values);
+ }
+
+ return row;
+ }
+
+ public InternalArray evolution(BinaryArray nullCounts, @Nullable Long
rowCount) {
+ if (indexMapping == null) {
+ return nullCounts;
+ }
+
+ if (rowCount == null) {
+ throw new RuntimeException("Schema Evolution for stats needs row
count.");
+ }
+
+ return new NullCountsEvoArray(indexMapping, nullCounts, rowCount);
+ }
+
public static RowType schema() {
List<DataField> fields = new ArrayList<>();
fields.add(new DataField(0, "_MIN_VALUES", newBytesType(false)));
@@ -134,4 +179,155 @@ public class FieldStatsArraySerializer {
rowType.getFieldNames().toArray(new String[0]))
.build();
}
+
+ private static class NullCountsEvoArray implements InternalArray {
+
+ private final int[] indexMapping;
+ private final InternalArray array;
+ private final long notFoundValue;
+
+ protected NullCountsEvoArray(int[] indexMapping, InternalArray array,
long notFoundValue) {
+ this.indexMapping = indexMapping;
+ this.array = array;
+ this.notFoundValue = notFoundValue;
+ }
+
+ @Override
+ public int size() {
+ return indexMapping.length;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ if (indexMapping[pos] < 0) {
+ return false;
+ }
+ return array.isNullAt(indexMapping[pos]);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ if (indexMapping[pos] < 0) {
+ return notFoundValue;
+ }
+ return array.getLong(indexMapping[pos]);
+ }
+
+ // ============================= Unsupported Methods
================================
+
+ @Override
+ public boolean getBoolean(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public short getShort(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getInt(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String toString() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean[] toBooleanArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public short[] toShortArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int[] toIntArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long[] toLongArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float[] toFloatArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public double[] toDoubleArray() {
+ throw new UnsupportedOperationException();
+ }
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
index b02772763..9c5f421ac 100644
---
a/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
+++
b/paimon-core/src/main/java/org/apache/paimon/stats/FieldStatsConverters.java
@@ -19,16 +19,21 @@
package org.apache.paimon.stats;
import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.schema.IndexCastMapping;
import org.apache.paimon.schema.SchemaEvolutionUtil;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
+import javax.annotation.Nullable;
+
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
+import static
org.apache.paimon.schema.SchemaEvolutionUtil.createIndexCastMapping;
+
/** Converters to create field stats array serializer. */
public class FieldStatsConverters {
private final Function<Long, List<DataField>> schemaFields;
@@ -57,15 +62,19 @@ public class FieldStatsConverters {
List<DataField> schemaTableFields =
tableFields.updateAndGet(v -> v == null ?
tableDataFields : v);
List<DataField> dataFields = schemaFields.apply(id);
- int[] indexMapping =
-
SchemaEvolutionUtil.createIndexMapping(schemaTableFields, dataFields);
+ IndexCastMapping indexCastMapping =
+ createIndexCastMapping(schemaTableFields,
dataFields);
+ @Nullable int[] indexMapping =
indexCastMapping.getIndexMapping();
CastExecutor<Object, Object>[] castExecutors =
(CastExecutor<Object, Object>[])
SchemaEvolutionUtil.createConvertMapping(
schemaTableFields, dataFields,
indexMapping);
// Create field stats array serializer with schema
evolution
return new FieldStatsArraySerializer(
- new RowType(dataFields), indexMapping,
castExecutors);
+ new RowType(dataFields),
+ indexMapping,
+ castExecutors,
+ indexCastMapping.getCastMapping());
});
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
index f0508bda8..0949de2df 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java
@@ -372,7 +372,7 @@ public class FilesTable implements ReadonlyTable {
fieldStatsConverters.getOrCreate(file.schemaId());
// Create value stats
FieldStats[] fieldStatsArray =
- tableStats.fields(fieldStatsArraySerializer,
file.rowCount());
+ fieldStatsArraySerializer.fromBinary(tableStats,
file.rowCount());
lazyNullValueCounts = new TreeMap<>();
lazyLowerValueBounds = new TreeMap<>();
lazyUpperValueBounds = new TreeMap<>();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
b/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
index 04a47c65b..36aeb6d6f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java
@@ -44,7 +44,7 @@ public class BulkFormatMapping {
private final FormatReaderFactory bulkFormat;
public BulkFormatMapping(
- int[] indexMapping,
+ @Nullable int[] indexMapping,
@Nullable CastFieldGetter[] castMapping,
@Nullable Pair<int[], RowType> partitionPair,
FormatReaderFactory bulkFormat) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
index e18eaf066..163ba3bec 100644
---
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
@@ -567,15 +567,20 @@ public class AppendOnlyWriterTest {
STATS_SERIALIZER.toBinary(
new FieldStats[] {
initStats(
-
toCompact.get(0).valueStats().min().getInt(0),
- toCompact.get(size -
1).valueStats().max().getInt(0),
+
toCompact.get(0).valueStats().minValues().getInt(0),
+ toCompact.get(size -
1).valueStats().maxValues().getInt(0),
0),
initStats(
-
toCompact.get(0).valueStats().min().getString(1).toString(),
+ toCompact
+ .get(0)
+ .valueStats()
+ .minValues()
+ .getString(1)
+ .toString(),
toCompact
.get(size - 1)
.valueStats()
- .max()
+ .maxValues()
.getString(1)
.toString(),
0),
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 5be026b7c..cdd16a377 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -19,6 +19,7 @@
package org.apache.paimon.manifest;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.format.FieldStats;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileIOFinder;
@@ -26,6 +27,7 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
import org.apache.paimon.stats.StatsTestUtils;
import org.apache.paimon.utils.FailingFileIO;
import org.apache.paimon.utils.FileStorePathFactory;
@@ -128,12 +130,14 @@ public class ManifestFileTest {
.isEqualTo(expected.numDeletedFiles());
// check stats
- for (int i = 0; i < expected.partitionStats().fields(null).length;
i++) {
+ FieldStatsArraySerializer statsConverter = new
FieldStatsArraySerializer(DEFAULT_PART_TYPE);
+ FieldStats[] fieldStats =
statsConverter.fromBinary(expected.partitionStats());
+ for (int i = 0; i < fieldStats.length; i++) {
int idx = i;
StatsTestUtils.checkRollingFileStats(
- expected.partitionStats().fields(null)[i],
+ fieldStats[i],
actual,
- meta -> meta.partitionStats().fields(null)[idx]);
+ meta ->
statsConverter.fromBinary(meta.partitionStats())[idx]);
}
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java
b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java
index 68a1a0265..f43610f71 100644
---
a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java
@@ -18,8 +18,10 @@
package org.apache.paimon.partition;
+import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.format.FieldStats;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
@@ -45,7 +47,8 @@ public class PartitionPredicateTest {
PartitionPredicate.fromMultiple(RowType.of(),
Collections.singletonList(EMPTY_ROW));
assertThat(predicate.test(EMPTY_ROW)).isTrue();
- assertThat(predicate.test(1, new FieldStats[] {})).isTrue();
+ assertThat(predicate.test(1, EMPTY_ROW, EMPTY_ROW,
BinaryArray.fromLongArray(new Long[0])))
+ .isTrue();
}
@Test
@@ -150,7 +153,16 @@ public class PartitionPredicateTest {
}
private boolean validate(PartitionPredicate predicate, FieldStats[]
fieldStats) {
- return predicate.test(3, fieldStats);
+ Object[] min = new Object[fieldStats.length];
+ Object[] max = new Object[fieldStats.length];
+ Long[] nullCounts = new Long[fieldStats.length];
+ for (int i = 0; i < fieldStats.length; i++) {
+ min[i] = fieldStats[i].minValue();
+ max[i] = fieldStats[i].maxValue();
+ nullCounts[i] = fieldStats[i].nullCount();
+ }
+ return predicate.test(
+ 3, GenericRow.of(min), GenericRow.of(max),
BinaryArray.fromLongArray(nullCounts));
}
private static BinaryRow createPart(int i, int j) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/stats/BinaryTableStatsTest.java
b/paimon-core/src/test/java/org/apache/paimon/stats/BinaryTableStatsTest.java
deleted file mode 100644
index f4e0826e7..000000000
---
a/paimon-core/src/test/java/org/apache/paimon/stats/BinaryTableStatsTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.stats;
-
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryRowWriter;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.format.FieldStats;
-import org.apache.paimon.types.IntType;
-import org.apache.paimon.types.RowType;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Tests for {@link BinaryTableStats}. */
-public class BinaryTableStatsTest {
- @Test
- public void testBinaryTableStats() {
- List<Integer> minList = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9);
- List<Integer> maxList = Arrays.asList(11, 12, 13, 14, 15, 16, 17, 18,
19);
- Long[] nullCounts = new Long[] {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L};
-
- BinaryRow minRowData = binaryRow(minList);
- BinaryRow maxRowData = binaryRow(maxList);
- BinaryTableStats tableStats1 = new BinaryTableStats(minRowData,
maxRowData, nullCounts);
-
- InternalRow rowData = tableStats1.toRowData();
- BinaryTableStats tableStats2 = new BinaryTableStats(rowData);
- assertThat(tableStats1.min()).isEqualTo(tableStats2.min());
- assertThat(tableStats1.max()).isEqualTo(tableStats2.max());
-
assertThat(tableStats1.nullCounts()).isEqualTo(tableStats2.nullCounts());
- assertThat(tableStats1).isEqualTo(tableStats2);
-
- FieldStatsArraySerializer serializer =
- new FieldStatsArraySerializer(
- RowType.of(
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType(),
- new IntType()));
-
- FieldStats[] fieldStatsArray1 = tableStats1.fields(serializer, 100L);
- FieldStats[] fieldStatsArray2 = tableStats2.fields(serializer, 100L);
-
assertThat(fieldStatsArray1.length).isEqualTo(fieldStatsArray2.length).isEqualTo(9);
- assertThat(fieldStatsArray1).isEqualTo(fieldStatsArray2);
- for (int i = 0; i < fieldStatsArray1.length; i++) {
- FieldStats fieldStats = fieldStatsArray1[i];
- assertThat(fieldStats.minValue()).isEqualTo(1 + i);
- assertThat(fieldStats.maxValue()).isEqualTo(11 + i);
- assertThat(fieldStats.nullCount()).isEqualTo(i);
- }
-
- BinaryTableStats tableStats3 =
- new BinaryTableStats(minRowData, maxRowData, nullCounts,
fieldStatsArray1);
- assertThat(tableStats3).isEqualTo(tableStats1).isEqualTo(tableStats2);
- assertThat(tableStats3.fields(serializer)).isEqualTo(fieldStatsArray2);
- }
-
- private BinaryRow binaryRow(List<Integer> valueList) {
- BinaryRow b = new BinaryRow(valueList.size());
- BinaryRowWriter writer = new BinaryRowWriter(b);
- for (int i = 0; i < valueList.size(); i++) {
- writer.writeInt(i, valueList.get(i));
- }
- writer.complete();
- return b;
- }
-}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
index 0eeaec583..2da96070f 100644
---
a/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/stats/FieldStatsArraySerializerTest.java
@@ -19,8 +19,10 @@
package org.apache.paimon.stats;
import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.schema.IndexCastMapping;
import org.apache.paimon.schema.SchemaEvolutionUtil;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.types.DataField;
@@ -66,21 +68,27 @@ public class FieldStatsArraySerializerTest {
Collections.EMPTY_MAP,
"");
- int[] indexMapping =
- SchemaEvolutionUtil.createIndexMapping(tableSchema.fields(),
dataSchema.fields());
+ IndexCastMapping indexCastMapping =
+ SchemaEvolutionUtil.createIndexCastMapping(
+ tableSchema.fields(), dataSchema.fields());
+ int[] indexMapping = indexCastMapping.getIndexMapping();
CastExecutor<Object, Object>[] converterMapping =
(CastExecutor<Object, Object>[])
SchemaEvolutionUtil.createConvertMapping(
tableSchema.fields(), dataSchema.fields(),
indexMapping);
FieldStatsArraySerializer fieldStatsArraySerializer =
new FieldStatsArraySerializer(
- tableSchema.logicalRowType(), indexMapping,
converterMapping);
+ tableSchema.logicalRowType(),
+ indexMapping,
+ converterMapping,
+ indexCastMapping.getCastMapping());
BinaryRow minRowData = row(1, 2, 3, 4);
BinaryRow maxRowData = row(100, 99, 98, 97);
Long[] nullCounts = new Long[] {1L, 0L, 10L, 100L};
- BinaryTableStats dataTableStats = new BinaryTableStats(minRowData,
maxRowData, nullCounts);
+ BinaryTableStats dataTableStats =
+ new BinaryTableStats(minRowData, maxRowData,
BinaryArray.fromLongArray(nullCounts));
- FieldStats[] fieldStatsArray =
dataTableStats.fields(fieldStatsArraySerializer, 1000L);
+ FieldStats[] fieldStatsArray =
fieldStatsArraySerializer.fromBinary(dataTableStats, 1000L);
assertThat(fieldStatsArray.length).isEqualTo(tableSchema.fields().size()).isEqualTo(5);
checkFieldStats(fieldStatsArray[0], 2, 99, 0L);
checkFieldStats(fieldStatsArray[1], 4, 97, 100L);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java
index ce98052e5..e63d481d9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java
@@ -73,18 +73,18 @@ public class StatsTableTest extends TableTestBase {
// should have partition stats
BinaryTableStats partitionStats = manifest.partitionStats();
- assertThat(partitionStats.min().getInt(0)).isEqualTo(1);
- assertThat(partitionStats.max().getInt(0)).isEqualTo(2);
+ assertThat(partitionStats.minValues().getInt(0)).isEqualTo(1);
+ assertThat(partitionStats.maxValues().getInt(0)).isEqualTo(2);
// should not have record stats because of NONE mode
ManifestFile manifestFile = store.manifestFileFactory().create();
DataFileMeta file =
manifestFile.read(manifest.fileName()).get(0).file();
BinaryTableStats recordStats = file.valueStats();
- assertThat(recordStats.min().isNullAt(0)).isTrue();
- assertThat(recordStats.min().isNullAt(1)).isTrue();
- assertThat(recordStats.min().isNullAt(2)).isTrue();
- assertThat(recordStats.max().isNullAt(0)).isTrue();
- assertThat(recordStats.max().isNullAt(1)).isTrue();
- assertThat(recordStats.max().isNullAt(2)).isTrue();
+ assertThat(recordStats.minValues().isNullAt(0)).isTrue();
+ assertThat(recordStats.minValues().isNullAt(1)).isTrue();
+ assertThat(recordStats.minValues().isNullAt(2)).isTrue();
+ assertThat(recordStats.maxValues().isNullAt(0)).isTrue();
+ assertThat(recordStats.maxValues().isNullAt(1)).isTrue();
+ assertThat(recordStats.maxValues().isNullAt(2)).isTrue();
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
index a95bd63b6..f1fc1f958 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java
@@ -20,18 +20,23 @@ package org.apache.paimon.table;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.Decimal;
-import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.BinaryTableStats;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
+import org.apache.paimon.stats.FieldStatsConverters;
import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.DataField;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@@ -91,15 +96,15 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
protected void validateStatsField(List<DataFileMeta> fileMetaList) {
for (DataFileMeta fileMeta : fileMetaList) {
- FieldStats[] statsArray =
getTableValueStats(fileMeta).fields(null);
- assertThat(statsArray.length).isEqualTo(12);
+ BinaryTableStats stats = getTableValueStats(fileMeta);
+ assertThat(stats.minValues().getFieldCount()).isEqualTo(12);
for (int i = 0; i < 11; i++) {
- assertThat(statsArray[i].minValue()).isNotNull();
- assertThat(statsArray[i].maxValue()).isNotNull();
+ assertThat(stats.minValues().isNullAt(i)).isFalse();
+ assertThat(stats.maxValues().isNullAt(i)).isFalse();
}
// Min and max value of binary type is null
- assertThat(statsArray[11].minValue()).isNull();
- assertThat(statsArray[11].maxValue()).isNull();
+ assertThat(stats.minValues().isNullAt(11)).isTrue();
+ assertThat(stats.maxValues().isNullAt(11)).isTrue();
}
}
@@ -165,7 +170,8 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
.collect(Collectors.toList()))
.containsAll(filesName);
- validateValuesWithNewSchema(filesName, fileMetaList);
+ validateValuesWithNewSchema(
+ schemas, table.schema().id(), filesName,
fileMetaList);
},
getPrimaryKeyNames(),
tableConfig,
@@ -216,7 +222,8 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
.containsAll(filesName);
// Compare all columns with table column type
- validateValuesWithNewSchema(filesName, fileMetaList);
+ validateValuesWithNewSchema(
+ schemas, table.schema().id(), filesName,
fileMetaList);
},
getPrimaryKeyNames(),
tableConfig,
@@ -224,14 +231,22 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
}
protected void validateValuesWithNewSchema(
- List<String> filesName, List<DataFileMeta> fileMetaList) {
+ Map<Long, TableSchema> tableSchemas,
+ long schemaId,
+ List<String> filesName,
+ List<DataFileMeta> fileMetaList) {
+ Function<Long, List<DataField>> schemaFields = id ->
tableSchemas.get(id).fields();
+ FieldStatsConverters converters = new
FieldStatsConverters(schemaFields, schemaId);
for (DataFileMeta fileMeta : fileMetaList) {
- FieldStats[] statsArray =
getTableValueStats(fileMeta).fields(null);
- assertThat(statsArray.length).isEqualTo(12);
+ BinaryTableStats stats = getTableValueStats(fileMeta);
+ FieldStatsArraySerializer serializer =
converters.getOrCreate(fileMeta.schemaId());
+ InternalRow min = serializer.evolution(stats.minValues());
+ InternalRow max = serializer.evolution(stats.maxValues());
+ assertThat(stats.minValues().getFieldCount()).isEqualTo(12);
if (filesName.contains(fileMeta.fileName())) {
- checkTwoValues(statsArray);
+ checkTwoValues(min, max);
} else {
- checkOneValue(statsArray);
+ checkOneValue(min, max);
}
}
}
@@ -245,41 +260,28 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
* <li>types: a->int, b->varchar[10], c->varchar[10], d->double, e->int,
f->decimal,g->float,
* h->double, i->decimal, j->date, k->date, l->varbinary
* </ul>
- *
- * @param statsArray the field stats
*/
- private void checkOneValue(FieldStats[] statsArray) {
-
assertThat(statsArray[0].minValue()).isEqualTo((statsArray[0].maxValue())).isEqualTo(2);
- assertThat(statsArray[1].minValue())
- .isEqualTo(statsArray[1].maxValue())
+ private void checkOneValue(InternalRow min, InternalRow max) {
+ assertThat(min.getInt(0)).isEqualTo(max.getInt(0)).isEqualTo(2);
+ assertThat(min.getString(1))
+ .isEqualTo(max.getString(1))
.isEqualTo(BinaryString.fromString("400"));
- assertThat(statsArray[2].minValue())
- .isEqualTo(statsArray[2].maxValue())
+ assertThat(min.getString(2))
+ .isEqualTo(max.getString(2))
.isEqualTo(BinaryString.fromString("401"));
- assertThat((Double) statsArray[3].minValue())
- .isEqualTo((Double) statsArray[3].maxValue())
- .isEqualTo(402D);
- assertThat((Integer) statsArray[4].minValue())
- .isEqualTo(statsArray[4].maxValue())
- .isEqualTo(403);
- assertThat(((Decimal)
statsArray[5].minValue()).toBigDecimal().intValue())
- .isEqualTo(((Decimal)
statsArray[5].maxValue()).toBigDecimal().intValue())
+
assertThat(min.getDouble(3)).isEqualTo(max.getDouble(3)).isEqualTo(402D);
+ assertThat(min.getInt(4)).isEqualTo(max.getInt(4)).isEqualTo(403);
+ assertThat(min.getDecimal(5, 10, 2).toBigDecimal().intValue())
+ .isEqualTo(max.getDecimal(5, 10, 2).toBigDecimal().intValue())
.isEqualTo(404);
- assertThat((Float) statsArray[6].minValue())
- .isEqualTo(statsArray[6].maxValue())
- .isEqualTo(405F);
- assertThat((Double) statsArray[7].minValue())
- .isEqualTo(statsArray[7].maxValue())
- .isEqualTo(406D);
- assertThat(((Decimal)
statsArray[8].minValue()).toBigDecimal().doubleValue())
- .isEqualTo(((Decimal)
statsArray[8].maxValue()).toBigDecimal().doubleValue())
+ assertThat(min.getFloat(6)).isEqualTo(max.getFloat(6)).isEqualTo(405F);
+
assertThat(min.getDouble(7)).isEqualTo(max.getDouble(7)).isEqualTo(406D);
+ assertThat(min.getDecimal(8, 10, 2).toBigDecimal().doubleValue())
+ .isEqualTo(max.getDecimal(8, 10,
2).toBigDecimal().doubleValue())
.isEqualTo(407D);
-
assertThat(statsArray[9].minValue()).isEqualTo(statsArray[9].maxValue()).isEqualTo(408);
-
assertThat(statsArray[10].minValue()).isEqualTo(statsArray[10].maxValue()).isEqualTo(409);
-
- // Min and max value of binary type is null
- assertThat(statsArray[11].minValue()).isNull();
- assertThat(statsArray[11].maxValue()).isNull();
+ assertThat(min.getInt(9)).isEqualTo(max.getInt(9)).isEqualTo(408);
+ assertThat(min.getInt(10)).isEqualTo(max.getInt(10)).isEqualTo(409);
+ assertThat(min.isNullAt(11)).isEqualTo(max.isNullAt(11)).isTrue();
}
/**
@@ -296,44 +298,43 @@ public abstract class ColumnTypeFileMetaTestBase extends
SchemaEvolutionTableTes
* f->decimal,g->float, h->double, i->decimal, j->date, k->date,
l->varbinary
* </ul>
*/
- private void checkTwoValues(FieldStats[] statsArray) {
- assertThat(statsArray[0].minValue()).isEqualTo(2);
- assertThat(statsArray[0].maxValue()).isEqualTo(2);
+ private void checkTwoValues(InternalRow min, InternalRow max) {
+ assertThat(min.getInt(0)).isEqualTo(2);
+ assertThat(max.getInt(0)).isEqualTo(2);
+
+ assertThat(min.getString(1)).isEqualTo(BinaryString.fromString("200
"));
+ assertThat(max.getString(1)).isEqualTo(BinaryString.fromString("300
"));
-
assertThat(statsArray[1].minValue()).isEqualTo(BinaryString.fromString("200
"));
-
assertThat(statsArray[1].maxValue()).isEqualTo(BinaryString.fromString("300
"));
+ assertThat(min.getString(2)).isEqualTo(BinaryString.fromString("201"));
+ assertThat(max.getString(2)).isEqualTo(BinaryString.fromString("301"));
-
assertThat(statsArray[2].minValue()).isEqualTo(BinaryString.fromString("201"));
-
assertThat(statsArray[2].maxValue()).isEqualTo(BinaryString.fromString("301"));
+ assertThat(min.getDouble(3)).isEqualTo(202D);
+ assertThat(max.getDouble(3)).isEqualTo(302D);
- assertThat((Double) statsArray[3].minValue()).isEqualTo(202D);
- assertThat((Double) statsArray[3].maxValue()).isEqualTo(302D);
+ assertThat(min.getInt(4)).isEqualTo(203);
+ assertThat(max.getInt(4)).isEqualTo(303);
- assertThat((Integer) statsArray[4].minValue()).isEqualTo(203);
- assertThat((Integer) statsArray[4].maxValue()).isEqualTo(303);
+ assertThat(min.getDecimal(5, 10,
2).toBigDecimal().intValue()).isEqualTo(204);
+ assertThat(max.getDecimal(5, 10,
2).toBigDecimal().intValue()).isEqualTo(304);
- assertThat(((Decimal)
statsArray[5].minValue()).toBigDecimal().intValue()).isEqualTo(204);
- assertThat(((Decimal)
statsArray[5].maxValue()).toBigDecimal().intValue()).isEqualTo(304);
+ assertThat(min.getFloat(6)).isEqualTo(205F);
+ assertThat(max.getFloat(6)).isEqualTo(305F);
- assertThat((Float) statsArray[6].minValue()).isEqualTo(205F);
- assertThat((Float) statsArray[6].maxValue()).isEqualTo(305F);
+ assertThat(min.getDouble(7)).isEqualTo(206D);
+ assertThat(max.getDouble(7)).isEqualTo(306D);
- assertThat((Double) statsArray[7].minValue()).isEqualTo(206D);
- assertThat((Double) statsArray[7].maxValue()).isEqualTo(306D);
+ assertThat(min.getDecimal(8, 10,
2).toBigDecimal().doubleValue()).isEqualTo(207D);
+ assertThat(max.getDecimal(8, 10,
2).toBigDecimal().doubleValue()).isEqualTo(307D);
- assertThat(((Decimal)
statsArray[8].minValue()).toBigDecimal().doubleValue())
- .isEqualTo(207D);
- assertThat(((Decimal)
statsArray[8].maxValue()).toBigDecimal().doubleValue())
- .isEqualTo(307D);
+ assertThat(min.getInt(9)).isEqualTo(208);
+ assertThat(max.getInt(9)).isEqualTo(308);
- assertThat(statsArray[9].minValue()).isEqualTo(208);
- assertThat(statsArray[9].maxValue()).isEqualTo(308);
- assertThat(statsArray[10].minValue()).isEqualTo(209);
- assertThat(statsArray[10].maxValue()).isEqualTo(309);
+ assertThat(min.getInt(10)).isEqualTo(209);
+ assertThat(max.getInt(10)).isEqualTo(309);
// Min and max value of binary type is null
- assertThat(statsArray[11].minValue()).isNull();
- assertThat(statsArray[11].maxValue()).isNull();
+ assertThat(min.isNullAt(11)).isTrue();
+ assertThat(max.isNullAt(11)).isTrue();
}
@Override
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java
index f2ab5b9a9..d460e854a 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java
@@ -18,18 +18,22 @@
package org.apache.paimon.table;
-import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.stats.BinaryTableStats;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
+import org.apache.paimon.stats.FieldStatsConverters;
import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.DataField;
import org.junit.jupiter.api.Test;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@@ -73,36 +77,38 @@ public abstract class FileMetaFilterTestBase extends
SchemaEvolutionTableTestBas
.collect(Collectors.toList()))
.containsAll(filesName);
+ Function<Long, List<DataField>> schemaFields = id ->
schemas.get(id).fields();
+ FieldStatsConverters converters =
+ new FieldStatsConverters(schemaFields,
table.schema().id());
for (DataFileMeta fileMeta : fileMetaList) {
- FieldStats[] statsArray =
getTableValueStats(fileMeta).fields(null);
- assertThat(statsArray.length).isEqualTo(6);
- if (filesName.contains(fileMeta.fileName())) {
- assertThat(statsArray[0].minValue()).isNotNull();
- assertThat(statsArray[0].maxValue()).isNotNull();
- assertThat(statsArray[1].minValue()).isNotNull();
- assertThat(statsArray[1].maxValue()).isNotNull();
- assertThat(statsArray[2].minValue()).isNotNull();
- assertThat(statsArray[2].maxValue()).isNotNull();
+ BinaryTableStats stats = getTableValueStats(fileMeta);
+ FieldStatsArraySerializer serializer =
+ converters.getOrCreate(fileMeta.schemaId());
+ InternalRow min =
serializer.evolution(stats.minValues());
+ InternalRow max =
serializer.evolution(stats.maxValues());
+
+ assertThat(min.getFieldCount()).isEqualTo(6);
- assertThat(statsArray[3].minValue()).isNull();
- assertThat(statsArray[3].maxValue()).isNull();
- assertThat(statsArray[4].minValue()).isNull();
- assertThat(statsArray[4].maxValue()).isNull();
- assertThat(statsArray[5].minValue()).isNull();
- assertThat(statsArray[5].maxValue()).isNull();
+ assertThat(min.isNullAt(0)).isFalse();
+ assertThat(max.isNullAt(0)).isFalse();
+ assertThat(min.isNullAt(1)).isFalse();
+ assertThat(max.isNullAt(1)).isFalse();
+ assertThat(min.isNullAt(2)).isFalse();
+ assertThat(max.isNullAt(2)).isFalse();
+ if (filesName.contains(fileMeta.fileName())) {
+ assertThat(min.isNullAt(3)).isTrue();
+ assertThat(max.isNullAt(3)).isTrue();
+ assertThat(min.isNullAt(4)).isTrue();
+ assertThat(max.isNullAt(4)).isTrue();
+ assertThat(min.isNullAt(5)).isTrue();
+ assertThat(max.isNullAt(5)).isTrue();
} else {
- assertThat(statsArray[0].minValue()).isNotNull();
- assertThat(statsArray[0].maxValue()).isNotNull();
- assertThat(statsArray[1].minValue()).isNotNull();
- assertThat(statsArray[1].maxValue()).isNotNull();
- assertThat(statsArray[2].minValue()).isNotNull();
- assertThat(statsArray[2].maxValue()).isNotNull();
- assertThat(statsArray[3].minValue()).isNotNull();
- assertThat(statsArray[3].maxValue()).isNotNull();
- assertThat(statsArray[4].minValue()).isNotNull();
- assertThat(statsArray[4].maxValue()).isNotNull();
- assertThat(statsArray[5].minValue()).isNotNull();
- assertThat(statsArray[5].maxValue()).isNotNull();
+ assertThat(min.isNullAt(3)).isFalse();
+ assertThat(max.isNullAt(3)).isFalse();
+ assertThat(min.isNullAt(4)).isFalse();
+ assertThat(max.isNullAt(4)).isFalse();
+ assertThat(min.isNullAt(5)).isFalse();
+ assertThat(max.isNullAt(5)).isFalse();
}
}
},
@@ -177,12 +183,19 @@ public abstract class FileMetaFilterTestBase extends
SchemaEvolutionTableTestBas
List<DataSplit> allSplits =
table.newSnapshotReader().read().dataSplits();
assertThat(filterAllSplits).isEqualTo(allSplits);
+ Function<Long, List<DataField>> schemaFields = id ->
schemas.get(id).fields();
+ FieldStatsConverters converters =
+ new FieldStatsConverters(schemaFields,
table.schema().id());
Set<String> filterFileNames = new HashSet<>();
for (DataSplit dataSplit : filterAllSplits) {
for (DataFileMeta dataFileMeta :
dataSplit.dataFiles()) {
- FieldStats[] fieldStats =
getTableValueStats(dataFileMeta).fields(null);
- int minValue = (Integer) fieldStats[1].minValue();
- int maxValue = (Integer) fieldStats[1].maxValue();
+ BinaryTableStats stats =
getTableValueStats(dataFileMeta);
+ FieldStatsArraySerializer serializer =
+
converters.getOrCreate(dataFileMeta.schemaId());
+ InternalRow min =
serializer.evolution(stats.minValues());
+ InternalRow max =
serializer.evolution(stats.maxValues());
+ int minValue = min.getInt(1);
+ int maxValue = max.getInt(1);
if (minValue >= 14
&& minValue <= 19
&& maxValue >= 14
@@ -249,11 +262,18 @@ public abstract class FileMetaFilterTestBase extends
SchemaEvolutionTableTestBas
checkFilterRowCount(toDataFileMetas(allSplits), 12L);
Set<String> filterFileNames = new HashSet<>();
+ Function<Long, List<DataField>> schemaFields = id ->
schemas.get(id).fields();
+ FieldStatsConverters converters =
+ new FieldStatsConverters(schemaFields,
table.schema().id());
for (DataSplit dataSplit : allSplits) {
for (DataFileMeta dataFileMeta :
dataSplit.dataFiles()) {
- FieldStats[] fieldStats =
getTableValueStats(dataFileMeta).fields(null);
- Integer minValue = (Integer)
fieldStats[3].minValue();
- Integer maxValue = (Integer)
fieldStats[3].maxValue();
+ BinaryTableStats stats =
getTableValueStats(dataFileMeta);
+ FieldStatsArraySerializer serializer =
+
converters.getOrCreate(dataFileMeta.schemaId());
+ InternalRow min =
serializer.evolution(stats.minValues());
+ InternalRow max =
serializer.evolution(stats.maxValues());
+ Integer minValue = min.isNullAt(3) ? null :
min.getInt(3);
+ Integer maxValue = max.isNullAt(3) ? null :
max.getInt(3);
if (minValue != null
&& maxValue != null
&& minValue > 1120
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
index 8b3e3726f..4edbd35a2 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java
@@ -19,20 +19,24 @@
package org.apache.paimon.table;
import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.BinaryTableStats;
+import org.apache.paimon.stats.FieldStatsArraySerializer;
+import org.apache.paimon.stats.FieldStatsConverters;
import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.DataField;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@@ -65,11 +69,11 @@ public class PrimaryKeyTableColumnTypeFileMetaTest extends
ColumnTypeFileMetaTes
@Override
protected void validateStatsField(List<DataFileMeta> fileMetaList) {
for (DataFileMeta fileMeta : fileMetaList) {
- FieldStats[] statsArray =
getTableValueStats(fileMeta).fields(null);
- assertThat(statsArray.length).isEqualTo(4);
+ BinaryTableStats stats = getTableValueStats(fileMeta);
+ assertThat(stats.maxValues().getFieldCount()).isEqualTo(4);
for (int i = 0; i < 4; i++) {
- assertThat(statsArray[i].minValue()).isNotNull();
- assertThat(statsArray[i].maxValue()).isNotNull();
+ assertThat(stats.minValues().isNullAt(i)).isFalse();
+ assertThat(stats.maxValues().isNullAt(i)).isFalse();
}
}
}
@@ -112,37 +116,40 @@ public class PrimaryKeyTableColumnTypeFileMetaTest
extends ColumnTypeFileMetaTes
/** We can only validate the values in primary keys for changelog with key
table. */
@Override
protected void validateValuesWithNewSchema(
- List<String> filesName, List<DataFileMeta> fileMetaList) {
+ Map<Long, TableSchema> tableSchemas,
+ long schemaId,
+ List<String> filesName,
+ List<DataFileMeta> fileMetaList) {
+ Function<Long, List<DataField>> schemaFields =
+ id ->
tableSchemas.get(id).logicalTrimmedPrimaryKeysType().getFields();
+ FieldStatsConverters converters = new
FieldStatsConverters(schemaFields, schemaId);
for (DataFileMeta fileMeta : fileMetaList) {
- FieldStats[] statsArray =
getTableValueStats(fileMeta).fields(null);
- assertThat(statsArray.length).isEqualTo(4);
+ BinaryTableStats stats = getTableValueStats(fileMeta);
+ FieldStatsArraySerializer serializer =
converters.getOrCreate(fileMeta.schemaId());
+ InternalRow min = serializer.evolution(stats.minValues());
+ InternalRow max = serializer.evolution(stats.maxValues());
+ assertThat(min.getFieldCount()).isEqualTo(4);
if (filesName.contains(fileMeta.fileName())) {
- assertThat(statsArray[0].minValue())
- .isEqualTo(BinaryString.fromString("200 "));
- assertThat(statsArray[0].maxValue())
- .isEqualTo(BinaryString.fromString("300 "));
+
assertThat(min.getString(0)).isEqualTo(BinaryString.fromString("200 "));
+
assertThat(max.getString(0)).isEqualTo(BinaryString.fromString("300 "));
-
assertThat(statsArray[1].minValue()).isEqualTo(BinaryString.fromString("201"));
-
assertThat(statsArray[1].maxValue()).isEqualTo(BinaryString.fromString("301"));
+
assertThat(min.getString(1)).isEqualTo(BinaryString.fromString("201"));
+
assertThat(max.getString(1)).isEqualTo(BinaryString.fromString("301"));
- assertThat((Double) statsArray[2].minValue()).isEqualTo(202D);
- assertThat((Double) statsArray[2].maxValue()).isEqualTo(302D);
+ assertThat(min.getDouble(2)).isEqualTo(202D);
+ assertThat(max.getDouble(2)).isEqualTo(302D);
- assertThat((Integer) statsArray[3].minValue()).isEqualTo(203);
- assertThat((Integer) statsArray[3].maxValue()).isEqualTo(303);
+ assertThat(min.getInt(3)).isEqualTo(203);
+ assertThat(max.getInt(3)).isEqualTo(303);
} else {
- assertThat(statsArray[0].minValue())
- .isEqualTo(statsArray[0].maxValue())
+ assertThat(min.getString(0))
+ .isEqualTo(max.getString(0))
.isEqualTo(BinaryString.fromString("400"));
- assertThat(statsArray[1].minValue())
- .isEqualTo(statsArray[1].maxValue())
+ assertThat(min.getString(1))
+ .isEqualTo(max.getString(1))
.isEqualTo(BinaryString.fromString("401"));
- assertThat((Double) statsArray[2].minValue())
- .isEqualTo((Double) statsArray[2].maxValue())
- .isEqualTo(402D);
- assertThat((Integer) statsArray[3].minValue())
- .isEqualTo(statsArray[3].maxValue())
- .isEqualTo(403);
+
assertThat(min.getDouble(2)).isEqualTo(max.getDouble(2)).isEqualTo(402D);
+
assertThat(min.getInt(3)).isEqualTo(max.getInt(3)).isEqualTo(403);
}
}
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateConverterTest.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateConverterTest.java
index e5df44849..e4f70ab13 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateConverterTest.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateConverterTest.java
@@ -21,6 +21,7 @@ package org.apache.paimon.flink;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.format.FieldStats;
+import org.apache.paimon.predicate.FieldStatsUtils;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
@@ -265,7 +266,11 @@ public class PredicateConverterTest {
IntStream.range(0, rowCountList.size())
.forEach(
i ->
- assertThat(predicate.test(rowCountList.get(i),
statsList.get(i)))
+ assertThat(
+ FieldStatsUtils.test(
+ predicate,
+ rowCountList.get(i),
+ statsList.get(i)))
.isEqualTo(expectedForStats.get(i)));
}
