This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 64384ef3a [core] Revert added_file_count and delete_file_count in
SnapshotsTable (#2941)
64384ef3a is described below
commit 64384ef3aea32b140cf970c41b8d61ea6bc1f2ad
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Mar 4 18:08:22 2024 +0800
[core] Revert added_file_count and delete_file_count in SnapshotsTable
(#2941)
---
docs/content/how-to/system-tables.md | 12 ++++++------
.../apache/paimon/table/system/SnapshotsTable.java | 22 +++++-----------------
.../paimon/table/system/SnapshotsTableTest.java | 11 ++---------
3 files changed, 13 insertions(+), 32 deletions(-)
diff --git a/docs/content/how-to/system-tables.md
b/docs/content/how-to/system-tables.md
index 2efecf070..8e5a10768 100644
--- a/docs/content/how-to/system-tables.md
+++ b/docs/content/how-to/system-tables.md
@@ -45,12 +45,12 @@ You can query the snapshot history information of the table
through snapshots ta
SELECT * FROM my_table$snapshots;
/*
-+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
-| snapshot_id | schema_id | commit_user | commit_identifier |
commit_kind | commit_time | base_manifest_list |
delta_manifest_list | changelog_manifest_list | total_record_count
| delta_record_count | changelog_record_count | added_file_count |
delete_file_count | watermark |
-+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
-| 2 | 0 | 7ca4cd28-98e... | 2 |
APPEND | 2022-10-26 11:44:15.600 | manifest-list-31323d5f-76e6... |
manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... |
2 | 2 | 0 | 2 |
0 | 1666755855600 |
-| 1 | 0 | 870062aa-3e9... | 1 |
APPEND | 2022-10-26 11:44:15.148 | manifest-list-31593d5f-76e6... |
manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... |
1 | 1 | 0 | 1 |
0 | 1666755855148 |
-+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
++--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+----------------+
+| snapshot_id | schema_id | commit_user | commit_identifier |
commit_kind | commit_time | base_manifest_list |
delta_manifest_list | changelog_manifest_list | total_record_count
| delta_record_count | changelog_record_count | watermark |
++--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+----------------+
+| 2 | 0 | 7ca4cd28-98e... | 2 |
APPEND | 2022-10-26 11:44:15.600 | manifest-list-31323d5f-76e6... |
manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... |
2 | 2 | 0 | 1666755855600 |
+| 1 | 0 | 870062aa-3e9... | 1 |
APPEND | 2022-10-26 11:44:15.148 | manifest-list-31593d5f-76e6... |
manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... |
1 | 1 | 0 | 1666755855148 |
++--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+-------------------------------
+--------------------------------+---------------------+---------------------+-------------------------+----------------+
2 rows in set
*/
```
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
index 944c8e605..ee9cff1d1 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
@@ -26,8 +26,6 @@ import org.apache.paimon.data.Timestamp;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.FileStoreTable;
@@ -40,7 +38,6 @@ import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.TimestampType;
import org.apache.paimon.utils.IteratorRecordReader;
@@ -96,9 +93,7 @@ public class SnapshotsTable implements ReadonlyTable {
new DataField(9, "total_record_count", new
BigIntType(true)),
new DataField(10, "delta_record_count", new
BigIntType(true)),
new DataField(11, "changelog_record_count", new
BigIntType(true)),
- new DataField(12, "added_file_count", new
IntType(true)),
- new DataField(13, "delete_file_count", new
IntType(true)),
- new DataField(14, "watermark", new
BigIntType(true))));
+ new DataField(12, "watermark", new
BigIntType(true))));
private final FileIO fileIO;
private final Path location;
@@ -133,7 +128,7 @@ public class SnapshotsTable implements ReadonlyTable {
@Override
public InnerTableRead newRead() {
- return new SnapshotsRead(fileIO, dataTable);
+ return new SnapshotsRead(fileIO);
}
@Override
@@ -199,11 +194,8 @@ public class SnapshotsTable implements ReadonlyTable {
private final FileIO fileIO;
private int[][] projection;
- private final FileStoreTable dataTable;
-
- public SnapshotsRead(FileIO fileIO, FileStoreTable dataTable) {
+ public SnapshotsRead(FileIO fileIO) {
this.fileIO = fileIO;
- this.dataTable = dataTable;
}
@Override
@@ -230,8 +222,7 @@ public class SnapshotsTable implements ReadonlyTable {
}
Path location = ((SnapshotsSplit) split).location;
Iterator<Snapshot> snapshots = new SnapshotManager(fileIO,
location).snapshots();
- Iterator<InternalRow> rows =
- Iterators.transform(snapshots, snapshot -> toRow(snapshot,
dataTable));
+ Iterator<InternalRow> rows = Iterators.transform(snapshots,
this::toRow);
if (projection != null) {
rows =
Iterators.transform(
@@ -240,8 +231,7 @@ public class SnapshotsTable implements ReadonlyTable {
return new IteratorRecordReader<>(rows);
}
- private InternalRow toRow(Snapshot snapshot, FileStoreTable dataTable)
{
- FileStoreScan.Plan plan =
dataTable.store().newScan().withSnapshot(snapshot).plan();
+ private InternalRow toRow(Snapshot snapshot) {
return GenericRow.of(
snapshot.id(),
snapshot.schemaId(),
@@ -258,8 +248,6 @@ public class SnapshotsTable implements ReadonlyTable {
snapshot.totalRecordCount(),
snapshot.deltaRecordCount(),
snapshot.changelogRecordCount(),
- plan.files(FileKind.ADD).size(),
- plan.files(FileKind.DELETE).size(),
snapshot.watermark());
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
index 4bb5f462e..fa89d3fca 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
@@ -29,8 +29,6 @@ import org.apache.paimon.data.Timestamp;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.SchemaUtils;
@@ -56,8 +54,6 @@ import static org.assertj.core.api.Assertions.assertThat;
public class SnapshotsTableTest extends TableTestBase {
private static final String tableName = "MyTable";
- private FileStoreTable table;
- private FileStoreScan scan;
private SnapshotsTable snapshotsTable;
private SnapshotManager snapshotManager;
@@ -78,8 +74,8 @@ public class SnapshotsTableTest extends TableTestBase {
snapshotManager = new SnapshotManager(fileIO, tablePath);
TableSchema tableSchema =
SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath),
schema);
- table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath,
tableSchema);
- scan = table.store().newScan();
+ FileStoreTable table =
+ FileStoreTableFactory.create(LocalFileIO.create(), tablePath,
tableSchema);
Identifier filesTableId =
identifier(tableName + Catalog.SYSTEM_TABLE_SPLITTER +
SnapshotsTable.SNAPSHOTS);
@@ -102,7 +98,6 @@ public class SnapshotsTableTest extends TableTestBase {
private List<InternalRow> getExceptedResult(long[] snapshotIds) {
List<InternalRow> expectedRow = new ArrayList<>();
for (long snapshotId : snapshotIds) {
- FileStoreScan.Plan plan = scan.withSnapshot(snapshotId).plan();
Snapshot snapshot = snapshotManager.snapshot(snapshotId);
expectedRow.add(
GenericRow.of(
@@ -121,8 +116,6 @@ public class SnapshotsTableTest extends TableTestBase {
snapshot.totalRecordCount(),
snapshot.deltaRecordCount(),
snapshot.changelogRecordCount(),
- plan.files(FileKind.ADD).size(),
- plan.files(FileKind.DELETE).size(),
snapshot.watermark()));
}
