This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7a08c89bc [core] Support statistic with time travel (#4251)
7a08c89bc is described below
commit 7a08c89bc0b973edd34ac58c94d4bcbb995766ba
Author: xuzifu666 <[email protected]>
AuthorDate: Tue Oct 8 11:17:43 2024 +0800
[core] Support statistic with time travel (#4251)
---
docs/content/maintenance/system-tables.md | 16 ++++++++
.../paimon/table/AbstractFileStoreTable.java | 14 ++++++-
.../apache/paimon/table/system/StatisticTable.java | 2 +-
.../paimon/spark/sql/AnalyzeTableTestBase.scala | 44 ++++++++++++++++++++++
4 files changed, 73 insertions(+), 3 deletions(-)
diff --git a/docs/content/maintenance/system-tables.md
b/docs/content/maintenance/system-tables.md
index f19c75756..9c8bfc295 100644
--- a/docs/content/maintenance/system-tables.md
+++ b/docs/content/maintenance/system-tables.md
@@ -369,3 +369,19 @@ SELECT * FROM sys.catalog_options;
*/
```
+### Statistic Table
+You can query the statistic information through statistic table.
+
+```sql
+SELECT * FROM T$statistics;
+
+/*
++--------------+------------+-----------------------+------------------+----------+
+| snapshot_id | schema_id | mergedRecordCount | mergedRecordSize |
colstat |
++--------------+------------+-----------------------+------------------+----------+
+| 2 | 0 | 2 | 2 | {}
|
++--------------+------------+-----------------------+------------------+----------+
+1 rows in set
+*/
+```
+
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index 171b7325a..7682cdf77 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -168,8 +168,18 @@ abstract class AbstractFileStoreTable implements
FileStoreTable {
@Override
public Optional<Statistics> statistics() {
- // todo: support time travel
- Snapshot latestSnapshot = snapshotManager().latestSnapshot();
+ Snapshot latestSnapshot;
+ Long snapshotId = coreOptions().scanSnapshotId();
+ if (snapshotId == null) {
+ snapshotId = snapshotManager().latestSnapshotId();
+ }
+
+ if (snapshotId != null &&
snapshotManager().snapshotExists(snapshotId)) {
+ latestSnapshot = snapshotManager().snapshot(snapshotId);
+ } else {
+ latestSnapshot = snapshotManager().latestSnapshot();
+ }
+
if (latestSnapshot != null) {
return store().newStatsFileHandler().readStats(latestSnapshot);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java
index 71642aced..faaa654eb 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java
@@ -65,7 +65,7 @@ public class StatisticTable implements ReadonlyTable {
public static final String STATISTICS = "statistics";
- public static final RowType TABLE_TYPE =
+ private static final RowType TABLE_TYPE =
new RowType(
Arrays.asList(
new DataField(0, "snapshot_id", new
BigIntType(false)),
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
index be7579b20..21d43f77f 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
@@ -73,6 +73,50 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
Row(2, 0, 2, "{ }"))
}
+ test("Paimon analyze: test statistic system table with snapshot") {
+ spark.sql(s"""
+ |CREATE TABLE T (id STRING, name STRING, i INT, l LONG)
+ |USING PAIMON
+ |TBLPROPERTIES ('primary-key'='id')
+ |""".stripMargin)
+
+ spark.sql(s"INSERT INTO T VALUES ('1', 'a', 1, 1)")
+ spark.sql(s"INSERT INTO T VALUES ('2', 'aaa', 1, 2)")
+ Assertions.assertEquals(0, spark.sql("select * from
`T$statistics`").count())
+
+ spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS")
+
+ spark.sql(s"INSERT INTO T VALUES ('3', 'b', 2, 1)")
+ spark.sql(s"INSERT INTO T VALUES ('4', 'bbb', 3, 2)")
+
+ spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS")
+
+ withSQLConf("spark.paimon.scan.snapshot-id" -> "3") {
+ checkAnswer(
+ sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM
`T$statistics`"),
+ Row(2, 0, 2, "{ }"))
+ }
+
+ withSQLConf("spark.paimon.scan.snapshot-id" -> "4") {
+ checkAnswer(
+ sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM
`T$statistics`"),
+ Row(2, 0, 2, "{ }"))
+ }
+
+ withSQLConf("spark.paimon.scan.snapshot-id" -> "6") {
+ checkAnswer(
+ sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM
`T$statistics`"),
+ Row(5, 0, 4, "{ }"))
+ }
+
+ withSQLConf("spark.paimon.scan.snapshot-id" -> "100") {
+ checkAnswer(
+ sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM
`T$statistics`"),
+ Row(5, 0, 4, "{ }"))
+ }
+
+ }
+
test("Paimon analyze: analyze table without snapshot") {
spark.sql(s"CREATE TABLE T (id STRING, name STRING)")
spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS")
