This is an automated email from the ASF dual-hosted git repository. wzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 75947e254a800fbc168b054d488d529a907c4e63 Author: Gergely Fürnstáhl <[email protected]> AuthorDate: Mon Jul 17 14:23:25 2023 +0200 IMPALA-12072: Include snapshot id of Iceberg tables in query plan / profile Including the snapshot id of Iceberg tables for the Iceberg SCAN operators can be useful to enable replayable queries. Replayable queries are useful, so we can better investigate performance problems / bugs. Testing: - Updated planner tests, added e2e test for time travel Change-Id: Iee0b4967429ea733729ad8e44df32e3b24b88525 Reviewed-on: http://gerrit.cloudera.org:8080/20204 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../org/apache/impala/planner/IcebergScanNode.java | 21 ++- .../apache/impala/planner/IcebergScanPlanner.java | 17 +- .../java/org/apache/impala/util/IcebergUtil.java | 6 + .../org/apache/impala/planner/PlannerTest.java | 19 +- .../org/apache/impala/planner/PlannerTestBase.java | 9 +- .../java/org/apache/impala/testutil/TestUtils.java | 3 + .../iceberg-predicates-disabled-subsetting.test | 5 +- .../queries/PlannerTest/iceberg-predicates.test | 24 ++- .../queries/PlannerTest/iceberg-v2-delete.test | 42 +++++ .../PlannerTest/iceberg-v2-tables-hash-join.test | 116 +++++++++++++ .../queries/PlannerTest/iceberg-v2-tables.test | 192 ++++++++++++++++++++- .../queries/PlannerTest/iceberg-v2-update.test | 42 +++++ .../queries/PlannerTest/insert-sort-by-zorder.test | 2 + .../queries/PlannerTest/tablesample.test | 11 +- tests/query_test/test_iceberg.py | 25 +++ 15 files changed, 506 insertions(+), 28 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java index a5abefea0..90cce6edb 100644 --- a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java +++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java @@ -70,6 +70,9 @@ public class IcebergScanNode extends HdfsScanNode { // already applied them and they won't filter any further rows. private List<Expr> skippedConjuncts_; + // The Iceberg snapshot id used for this scan. + private final long snapshotId_; + // This member is set when this scan node is the left child of an IcebergDeleteNode or // in other words when this scan node reads data files that have delete files // associated. Holds the scan node ID of the right child of the IcebergDeleteNode @@ -78,15 +81,16 @@ public class IcebergScanNode extends HdfsScanNode { public IcebergScanNode(PlanNodeId id, TableRef tblRef, List<Expr> conjuncts, MultiAggregateInfo aggInfo, List<FileDescriptor> fileDescs, - List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts) + List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, long snapshotId) throws ImpalaRuntimeException { this(id, tblRef, conjuncts, aggInfo, fileDescs, nonIdentityConjuncts, - skippedConjuncts, null); + skippedConjuncts, null, snapshotId); } public IcebergScanNode(PlanNodeId id, TableRef tblRef, List<Expr> conjuncts, MultiAggregateInfo aggInfo, List<FileDescriptor> fileDescs, - List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, PlanNodeId deleteId) + List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, PlanNodeId deleteId, + long snapshotId) throws ImpalaRuntimeException { super(id, tblRef.getDesc(), conjuncts, getIcebergPartition(((FeIcebergTable)tblRef.getTable()).getFeFsTable()), tblRef, @@ -102,6 +106,7 @@ public class IcebergScanNode extends HdfsScanNode { filesAreSorted_ = true; } nonIdentityConjuncts_ = nonIdentityConjuncts; + snapshotId_ = snapshotId; //TODO IMPALA-11577: optimize file format counting boolean hasParquet = false; boolean hasOrc = false; @@ -263,10 +268,14 @@ public class IcebergScanNode extends HdfsScanNode { @Override protected String getDerivedExplainString( String indentPrefix, TExplainLevel detailLevel) { + StringBuilder output = new StringBuilder(); + output.append( + indentPrefix + "Iceberg snapshot id: " + String.valueOf(snapshotId_) + "\n"); if (!skippedConjuncts_.isEmpty()) { - return indentPrefix + String.format("skipped Iceberg predicates: %s\n", - Expr.getExplainString(skippedConjuncts_, detailLevel)); + output.append(indentPrefix + + String.format("skipped Iceberg predicates: %s\n", + Expr.getExplainString(skippedConjuncts_, detailLevel))); } - return ""; + return output.toString(); } } diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java index 13b0251f4..9b54fbad1 100644 --- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java +++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java @@ -129,6 +129,8 @@ public class IcebergScanPlanner { private long dataFilesWithDeletesMaxPath_ = 0; private Set<Long> equalityDeleteSequenceNumbers_ = new HashSet<>(); + private final long snapshotId_; + public IcebergScanPlanner(Analyzer analyzer, PlannerContext ctx, TableRef iceTblRef, List<Expr> conjuncts, MultiAggregateInfo aggInfo) throws ImpalaException { @@ -140,6 +142,7 @@ public class IcebergScanPlanner { conjuncts_ = conjuncts; aggInfo_ = aggInfo; extractIcebergConjuncts(); + snapshotId_ = IcebergUtil.getSnapshotId(getIceTable(), tblRef_.getTimeTravelSpec()); } public PlanNode createIcebergScanPlan() throws ImpalaException { @@ -187,7 +190,7 @@ public class IcebergScanPlanner { Preconditions.checkState(dataFilesWithDeletes_.isEmpty()); PlanNode ret = new IcebergScanNode(ctx_.getNextNodeId(), tblRef_, conjuncts_, aggInfo_, dataFilesWithoutDeletes_, nonIdentityConjuncts_, - getSkippedConjuncts()); + getSkippedConjuncts(), snapshotId_); ret.init(analyzer_); return ret; } @@ -211,7 +214,7 @@ public class IcebergScanPlanner { // can just create a SCAN node for these and do a UNION ALL with the ANTI JOIN. IcebergScanNode dataScanNode = new IcebergScanNode( ctx_.getNextNodeId(), tblRef_, conjuncts_, aggInfo_, dataFilesWithoutDeletes_, - nonIdentityConjuncts_, getSkippedConjuncts()); + nonIdentityConjuncts_, getSkippedConjuncts(), snapshotId_); dataScanNode.init(analyzer_); List<Expr> outputExprs = tblRef_.getDesc().getSlots().stream().map( SlotRef::new).collect(Collectors.toList()); @@ -246,7 +249,7 @@ public class IcebergScanPlanner { addDeletePositionSlots(deleteDeltaRef); IcebergScanNode dataScanNode = new IcebergScanNode( dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_, - nonIdentityConjuncts_, getSkippedConjuncts(), deleteScanNodeId); + nonIdentityConjuncts_, getSkippedConjuncts(), deleteScanNodeId, snapshotId_); dataScanNode.init(analyzer_); IcebergScanNode deleteScanNode = new IcebergScanNode( deleteScanNodeId, @@ -255,7 +258,8 @@ public class IcebergScanPlanner { aggInfo_, Lists.newArrayList(positionDeleteFiles_), Collections.emptyList(), /*nonIdentityConjuncts*/ - Collections.emptyList()); /*skippedConjuncts*/ + Collections.emptyList(), /*skippedConjuncts*/ + snapshotId_); deleteScanNode.init(analyzer_); // Now let's create the JOIN node @@ -435,7 +439,7 @@ public class IcebergScanPlanner { PlanNodeId dataScanNodeId = ctx_.getNextNodeId(); IcebergScanNode dataScanNode = new IcebergScanNode( dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_, - nonIdentityConjuncts_, getSkippedConjuncts()); + nonIdentityConjuncts_, getSkippedConjuncts(), snapshotId_); addSlotsForEqualityDelete(equalityIds_, tblRef_); dataScanNode.init(analyzer_); @@ -464,7 +468,8 @@ public class IcebergScanPlanner { aggInfo_, Lists.newArrayList(equalityDeleteFiles_), Collections.emptyList(), /*nonIdentityConjuncts*/ - Collections.emptyList()); /*skippedConjuncts*/ + Collections.emptyList(), /*skippedConjuncts*/ + snapshotId_); deleteScanNode.init(analyzer_); Pair<List<BinaryPredicate>, List<Expr>> equalityJoinConjuncts = diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java index f0dba7713..ad2ec5401 100644 --- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java +++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java @@ -646,6 +646,12 @@ public class IcebergUtil { } } + public static long getSnapshotId(FeIcebergTable table, TimeTravelSpec timeTravelSpec) { + if (timeTravelSpec == null) return table.snapshotId(); + TableScan scan = createScanAsOf(table, timeTravelSpec); + return scan.snapshot().snapshotId(); + } + private static TableScan createScanAsOf(FeIcebergTable table, TimeTravelSpec timeTravelSpec) { TableScan scan = table.getIcebergApiTable().newScan(); diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java index b70eb86dd..8a3d077cc 100644 --- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java +++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java @@ -965,7 +965,8 @@ public class PlannerTest extends PlannerTestBase { TQueryOptions options = defaultQueryOptions(); runPlannerTestFile("tablesample", options, ImmutableSet.of(PlannerTestOption.EXTENDED_EXPLAIN, - PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS)); + PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS, + PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)); } @Test @@ -1283,7 +1284,8 @@ public class PlannerTest extends PlannerTestBase { TQueryOptions queryOptions = new TQueryOptions(); queryOptions.setIceberg_predicate_pushdown_subsetting(false); runPlannerTestFile("iceberg-predicates-disabled-subsetting", "functional_parquet", - queryOptions, ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY)); + queryOptions, ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY, + PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)); } /** @@ -1291,8 +1293,11 @@ public class PlannerTest extends PlannerTestBase { */ @Test public void testIcebergV2TableScans() { - runPlannerTestFile("iceberg-v2-tables", "functional_parquet", - ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY)); + TQueryOptions options = defaultQueryOptions(); + options.setTimezone("UTC"); + runPlannerTestFile("iceberg-v2-tables", "functional_parquet", options, + ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY, + PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)); } /** @@ -1303,7 +1308,8 @@ public class PlannerTest extends PlannerTestBase { TQueryOptions options = defaultQueryOptions(); options.setDisable_optimized_iceberg_v2_read(true); runPlannerTestFile("iceberg-v2-tables-hash-join", "functional_parquet", options, - ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY)); + ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY, + PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)); } /** @@ -1312,7 +1318,8 @@ public class PlannerTest extends PlannerTestBase { @Test public void testIcebergV2Delete() { runPlannerTestFile("iceberg-v2-delete", "functional_parquet", - ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY)); + ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY, + PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)); } @Test diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java index 53205334f..023a6147c 100644 --- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java +++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java @@ -590,6 +590,9 @@ public class PlannerTestBase extends FrontendTestBase { PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS)) { resultFilters.add(TestUtils.PARTITIONS_FILTER); } + if (!testOptions.contains(PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)) { + resultFilters.add(TestUtils.ICEBERG_SNAPSHOT_ID_FILTER); + } String planDiff = TestUtils.compareOutput( Lists.newArrayList(explainStr.split("\n")), expectedPlan, true, resultFilters); @@ -910,7 +913,11 @@ public class PlannerTestBase extends FrontendTestBase { DISABLE_HDFS_NUM_ROWS_ESTIMATE, // If set, make no attempt to validate the estimated number of rows for any // partitions in an hdfs table. - DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS + DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS, + // Verify that the snapshot ids in the plan match to the expected values. We + // can only do this for tests that operate on pre-written Iceberg tables, + // e.g. functional_parquet.iceberg_partitioned. + VALIDATE_ICEBERG_SNAPSHOT_IDS } protected void runPlannerTestFile(String testFile, TQueryOptions options) { diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java index 24582f8ad..761a2ffc5 100644 --- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java +++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java @@ -211,6 +211,9 @@ public class TestUtils { public static final IgnoreValueFilter CARDINALITY_FILTER = new IgnoreValueFilter("cardinality", "\\S+"); + public static final IgnoreValueFilter ICEBERG_SNAPSHOT_ID_FILTER = + new IgnoreValueFilter("Iceberg snapshot id", " \\d+", ':'); + // Ignore any values after 'rows=' in partitions: 0/24 rows=12.83K or // partitions: 0/24 rows=unavailable entries public static final ReplaceValueFilter PARTITIONS_FILTER = diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test index 5421100c1..2bcd45ace 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test @@ -8,6 +8,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.97KB predicates: `user` = 'Lisa', action = 'download' + Iceberg snapshot id: 8270633197658268308 row-size=24B cardinality=1 ==== # If no residual expression remain after Iceberg's planning, all partition-based @@ -18,6 +19,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.97KB + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'download' row-size=44B cardinality=6 ==== @@ -27,6 +29,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.97KB + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'download', event_time < TIMESTAMP '2022-01-01 00:00:00' row-size=44B cardinality=1 -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test index 8d22e0b32..9dc71186a 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test @@ -7,6 +7,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=730 size=1.25MB + Iceberg snapshot id: 188637238055891670 skipped Iceberg predicates: `year` = 2010 row-size=20B cardinality=7.30K ==== @@ -19,6 +20,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=124 size=216.63KB predicates: `month` = 1 + Iceberg snapshot id: 188637238055891670 row-size=24B cardinality=1.24K ==== # The predicates that couldn't be pushed to Iceberg are pushed down to the scan node, @@ -31,6 +33,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=730 size=1.25MB predicates: power(id, 3) > 1000 + Iceberg snapshot id: 188637238055891670 skipped Iceberg predicates: `year` = 2010 row-size=20B cardinality=730 ==== @@ -43,6 +46,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=730 size=1.25MB predicates: id > 1000 + Iceberg snapshot id: 188637238055891670 skipped Iceberg predicates: `year` = 2010 row-size=20B cardinality=730 ==== @@ -57,6 +61,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=2 size=3.49KB predicates: date_string_col = '061610' + Iceberg snapshot id: 188637238055891670 skipped Iceberg predicates: `year` = 2010 row-size=40B cardinality=2 ==== @@ -72,6 +77,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'click' row-size=8B cardinality=6 ==== @@ -85,6 +91,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: `user` = NULL + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'click' row-size=12B cardinality=1 ---- DISTRIBUTEDPLAN @@ -95,6 +102,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: `user` = NULL + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'click' row-size=12B cardinality=1 ==== @@ -106,6 +114,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=20 size=22.90KB predicates: action = NULL + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time < TIMESTAMP '2022-01-01 00:00:00' row-size=44B cardinality=2 ==== @@ -118,6 +127,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=9 size=10.33KB predicates: id < 10, action LIKE 'd%' + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time < TIMESTAMP '2022-01-01 00:00:00' row-size=44B cardinality=1 ==== @@ -129,6 +139,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=10 size=11.42KB predicates: (event_time < TIMESTAMP '2020-01-01 00:00:00' OR id > 10), action LIKE 'd%' + Iceberg snapshot id: 8270633197658268308 row-size=44B cardinality=1 ==== # Predicate on a partition introduced by partition evolution pushed down to the scan node and @@ -140,6 +151,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partition_evolution] HDFS partitions=1/1 files=62 size=108.30KB predicates: `month` = 12 + Iceberg snapshot id: 188637238055891670 skipped Iceberg predicates: `year` = 2010 row-size=40B cardinality=620 ==== @@ -153,6 +165,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=12 size=13.65KB predicates: action IN ('click', 'view'), (id > 5 OR (id < 2)) + Iceberg snapshot id: 8270633197658268308 row-size=44B cardinality=1 ==== # IS NOT NULL predicate skipped on partitioned column 'action', predicate on @@ -164,10 +177,10 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=9 size=10.33KB predicates: id < 10 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IS NOT NULL row-size=44B cardinality=1 ==== ----- QUERY # Regression test for IMPALA-12580: predicate filtering could throw UnsupportedOperationException select p_int from functional_parquet.iceberg_alltypes_part where i = 1 and p_int = i; ---- PLAN @@ -176,10 +189,10 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_alltypes_part] HDFS partitions=1/1 files=1 size=433B predicates: i = 1 + Iceberg snapshot id: 6167994413873848621 skipped Iceberg predicates: functional_parquet.iceberg_alltypes_part.p_int = 1 row-size=8B cardinality=1 ==== ----- QUERY # Regression test for IMPALA-12580: predicate filtering could throw UnsupportedOperationException select * from functional_parquet.iceberg_alltypes_part where i = 1 and p_bigint = 10 + i and p_int = p_bigint - 10; @@ -189,10 +202,10 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_alltypes_part] HDFS partitions=1/1 files=1 size=433B predicates: i = 1 + Iceberg snapshot id: 6167994413873848621 skipped Iceberg predicates: p_bigint = 11, p_int = 1 row-size=49B cardinality=1 ==== ----- QUERY select * from functional_parquet.iceberg_alltypes_part where p_string = concat(chr(cast(p_int + 104 as int)), "mpala") and p_string = concat("i", "mpala") and p_decimal between 122 and 124 and i = floor(p_float) @@ -202,10 +215,10 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_alltypes_part] HDFS partitions=1/1 files=1 size=433B predicates: i = floor(p_float), concat(chr(CAST(p_int + 104 AS INT)), 'mpala') = 'impala' + Iceberg snapshot id: 6167994413873848621 skipped Iceberg predicates: p_string = 'impala', p_decimal >= 122, p_decimal <= 124 row-size=49B cardinality=1 ==== ----- QUERY select * from functional_parquet.iceberg_alltypes_part where i in (1,2,3) and i = p_int; ---- PLAN PLAN-ROOT SINK @@ -213,6 +226,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_alltypes_part] HDFS partitions=1/1 files=1 size=433B predicates: i = p_int, i IN (1, 2, 3) + Iceberg snapshot id: 6167994413873848621 skipped Iceberg predicates: functional_parquet.iceberg_alltypes_part.p_int IN (1, 2, 3) row-size=49B cardinality=1 -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test index 6eac5b341..c75cdaea0 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test @@ -9,6 +9,7 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE] 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i = 3 + Iceberg snapshot id: 728158873687794725 row-size=24B cardinality=1 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE] @@ -20,6 +21,7 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE] 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i = 3 + Iceberg snapshot id: 728158873687794725 row-size=24B cardinality=1 ==== DELETE FROM iceberg_v2_delete_positional where id = 15 @@ -35,11 +37,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: id = 15 + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=1 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE] @@ -55,11 +59,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: id = 15 + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=1 ==== DELETE FROM iceberg_v2_delete_positional @@ -85,10 +91,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | | | |--04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete] | | HDFS partitions=1/1 files=1 size=1.54KB +| | Iceberg snapshot id: 5725822353600261755 | | row-size=182B cardinality=1 | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] | HDFS partitions=1/1 files=1 size=662B +| Iceberg snapshot id: 5725822353600261755 | row-size=28B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] @@ -96,11 +104,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B runtime filters: RF000 -> id + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=3 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE] @@ -133,10 +143,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | | | | | 04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete] | | HDFS partitions=1/1 files=1 size=1.54KB +| | Iceberg snapshot id: 5725822353600261755 | | row-size=182B cardinality=1 | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] | HDFS partitions=1/1 files=1 size=662B +| Iceberg snapshot id: 5725822353600261755 | row-size=28B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED] @@ -146,11 +158,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B runtime filters: RF000 -> id + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=3 ==== DELETE FROM iceberg_v2_delete_positional WHERE FILE__POSITION = id @@ -166,11 +180,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: FILE__POSITION = id + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=1 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE] @@ -186,11 +202,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: FILE__POSITION = id + Iceberg snapshot id: 5725822353600261755 row-size=28B cardinality=1 ==== delete from iceberg_v2_partitioned_position_deletes where id = 20; @@ -206,11 +224,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: id = 20 + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=2 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE] @@ -228,11 +248,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: id = 20 + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=2 ==== delete from iceberg_v2_partitioned_position_deletes where action = 'click'; @@ -248,10 +270,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.15KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.15KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'click' row-size=36B cardinality=6 ---- DISTRIBUTEDPLAN @@ -268,10 +292,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.15KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.15KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'click' row-size=36B cardinality=6 ==== @@ -288,11 +314,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: `user` LIKE 'A%' + Iceberg snapshot id: 8885697082976537578 row-size=48B cardinality=2 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE] @@ -310,11 +338,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: `user` LIKE 'A%' + Iceberg snapshot id: 8885697082976537578 row-size=48B cardinality=2 ==== delete from iceberg_v2_partitioned_position_deletes @@ -340,10 +370,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | |--04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete] | | HDFS partitions=1/1 files=1 size=1.54KB +| | Iceberg snapshot id: 5725822353600261755 | | row-size=182B cardinality=1 | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] | HDFS partitions=1/1 files=1 size=662B +| Iceberg snapshot id: 5725822353600261755 | row-size=28B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] @@ -351,11 +383,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB runtime filters: RF000 -> id + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=20 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE] @@ -390,10 +424,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | | | 04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete] | | HDFS partitions=1/1 files=1 size=1.54KB +| | Iceberg snapshot id: 5725822353600261755 | | row-size=182B cardinality=1 | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] | HDFS partitions=1/1 files=1 size=662B +| Iceberg snapshot id: 5725822353600261755 | row-size=28B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED] @@ -403,11 +439,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB runtime filters: RF000 -> id + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=20 ==== DELETE FROM iceberg_v2_partitioned_position_deletes WHERE FILE__POSITION = id @@ -423,11 +461,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: FILE__POSITION = id + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=2 ---- DISTRIBUTEDPLAN DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE] @@ -445,10 +485,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes- | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB predicates: FILE__POSITION = id + Iceberg snapshot id: 8885697082976537578 row-size=40B cardinality=2 ==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test index 51b7b8c46..a43b98c5b 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test @@ -23,6 +23,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i > 1 + Iceberg snapshot id: 728158873687794725 row-size=4B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -40,6 +41,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i > 1 + Iceberg snapshot id: 728158873687794725 row-size=4B cardinality=1 ==== SELECT count(*) from iceberg_v2_delete_positional; @@ -55,10 +57,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=20B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -80,10 +84,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=20B cardinality=3 ==== SELECT * from iceberg_v2_delete_positional; @@ -95,10 +101,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -112,10 +120,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ==== SELECT * from iceberg_v2_positional_delete_all_rows; @@ -127,10 +137,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -144,10 +156,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_no_deletes limit 1 @@ -156,6 +170,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 728158873687794725 limit: 1 row-size=16B cardinality=1 ---- DISTRIBUTEDPLAN @@ -166,6 +181,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 728158873687794725 limit: 1 row-size=16B cardinality=1 ==== @@ -179,10 +195,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -198,10 +216,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files limit 1 @@ -218,14 +238,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -245,16 +268,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files @@ -270,14 +296,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -295,16 +324,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 7490459762454857930; @@ -334,10 +366,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 | row-size=267B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 row-size=20B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -359,10 +393,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 | row-size=267B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 row-size=20B cardinality=3 ==== SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files; @@ -378,10 +414,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -403,12 +441,14 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 04:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ==== SELECT * from iceberg_v2_positional_update_all_rows @@ -424,14 +464,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | HDFS partitions=1/1 files=1 size=2.60KB +| | Iceberg snapshot id: 3877007445826010687 | | row-size=246B cardinality=3 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 3877007445826010687 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -449,14 +492,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | HDFS partitions=1/1 files=1 size=2.60KB +| | Iceberg snapshot id: 3877007445826010687 | | row-size=246B cardinality=3 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 3877007445826010687 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_partitioned_position_deletes @@ -468,10 +514,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=64B cardinality=20 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -485,12 +533,14 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 03:EXCHANGE [HASH(functional_parquet.iceberg_v2_partitioned_position_deletes.file__position,functional_parquet.iceberg_v2_partitioned_position_deletes.input__file__name)] | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=64B cardinality=20 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files @@ -507,16 +557,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i > 2 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i > 2 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -534,6 +587,7 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] @@ -541,11 +595,13 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i > 2 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i > 2 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ==== select * from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 1497619269847778439 @@ -560,6 +616,7 @@ PLAN-ROOT SINK | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 4363979609026842966 | row-size=16B cardinality=3 | 05:AGGREGATE [FINALIZE] @@ -575,14 +632,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -597,6 +657,7 @@ PLAN-ROOT SINK | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 4363979609026842966 | row-size=16B cardinality=3 | 11:AGGREGATE [FINALIZE] @@ -620,16 +681,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 08:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== with v as (select i + 1000 as ii, upper(s) as ss from iceberg_v2_positional_not_all_data_files_have_delete_files) @@ -646,16 +710,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i + 1000 > 1003 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i + 1000 > 1003 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -673,6 +740,7 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] @@ -680,11 +748,13 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i + 1000 > 1003 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i + 1000 > 1003 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ==== select * @@ -711,14 +781,17 @@ PLAN-ROOT SINK | | | | | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=24B cardinality=3 | | | 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=24B cardinality=3 | 04:UNION @@ -730,16 +803,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | runtime filters: RF000 -> i +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB runtime filters: RF000 -> i + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -774,14 +850,17 @@ PLAN-ROOT SINK | | | | | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=24B cardinality=3 | | | 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=24B cardinality=3 | 04:UNION @@ -795,6 +874,7 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 12:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] @@ -802,11 +882,13 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | runtime filters: RF000 -> i +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB runtime filters: RF000 -> i + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== select * from iceberg_v2_partitioned_position_deletes where action = 'download' and id > 0; @@ -818,11 +900,13 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: id > 0 + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ---- DISTRIBUTEDPLAN @@ -837,11 +921,13 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: id > 0 + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ==== @@ -855,11 +941,13 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: `user` = 'Lisa' + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ---- DISTRIBUTEDPLAN @@ -874,11 +962,13 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: `user` = 'Lisa' + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ==== @@ -889,6 +979,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -899,6 +990,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ==== @@ -909,6 +1001,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -919,6 +1012,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ==== @@ -929,6 +1023,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -939,6 +1034,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click') row-size=32B cardinality=1 ==== @@ -949,6 +1045,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -959,6 +1056,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view') row-size=32B cardinality=1 ==== @@ -969,6 +1067,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view')) row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -979,6 +1078,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view')) row-size=32B cardinality=1 ==== @@ -989,6 +1089,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=20 size=22.90KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a') row-size=32B cardinality=2 ---- DISTRIBUTEDPLAN @@ -999,6 +1100,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=20 size=22.90KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a') row-size=32B cardinality=2 ==== @@ -1009,6 +1111,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=0 size=0B + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00' row-size=28B cardinality=0 ---- DISTRIBUTEDPLAN @@ -1018,6 +1121,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=0 size=0B + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00' row-size=28B cardinality=0 ==== @@ -1031,10 +1135,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=6 ---- DISTRIBUTEDPLAN @@ -1049,10 +1155,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=6 ==== @@ -1074,10 +1182,12 @@ PLAN-ROOT SINK | | | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=20B cardinality=6 | 09:NESTED LOOP JOIN [CROSS JOIN] @@ -1096,10 +1206,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1126,12 +1238,14 @@ PLAN-ROOT SINK | | | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 16:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=20B cardinality=6 | 09:NESTED LOOP JOIN [CROSS JOIN, BROADCAST] @@ -1160,11 +1274,13 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 11:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)] | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test index 29578ffe2..22da72d9b 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test @@ -23,6 +23,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i > 1 + Iceberg snapshot id: 728158873687794725 row-size=4B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -40,6 +41,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i > 1 + Iceberg snapshot id: 728158873687794725 row-size=4B cardinality=1 ==== SELECT count(*) from iceberg_v2_delete_positional; @@ -55,10 +57,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=20B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -80,10 +84,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=20B cardinality=3 ==== SELECT * from iceberg_v2_delete_positional; @@ -95,10 +101,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -112,10 +120,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ==== SELECT * from iceberg_v2_positional_delete_all_rows; @@ -127,10 +137,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -144,10 +156,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_no_deletes limit 1 @@ -156,6 +170,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 728158873687794725 limit: 1 row-size=16B cardinality=1 ---- DISTRIBUTEDPLAN @@ -166,6 +181,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 728158873687794725 limit: 1 row-size=16B cardinality=1 ==== @@ -179,10 +195,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -198,10 +216,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete] | HDFS partitions=1/1 files=1 size=2.60KB +| Iceberg snapshot id: 444149380144800647 | row-size=246B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 444149380144800647 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files limit 1 @@ -218,14 +238,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -245,14 +268,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files @@ -268,14 +294,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -293,14 +322,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 7490459762454857930; @@ -330,10 +362,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 | row-size=267B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 row-size=20B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -355,10 +389,60 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 | row-size=267B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 + row-size=20B cardinality=3 +==== +SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_time as of "2022-08-19 13:50:00"; +---- PLAN +PLAN-ROOT SINK +| +03:AGGREGATE [FINALIZE] +| output: count(*) +| row-size=8B cardinality=1 +| +02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] +| row-size=20B cardinality=2 +| +|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] +| HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 +| row-size=267B cardinality=1 +| +00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] + HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 + row-size=20B cardinality=3 +---- DISTRIBUTEDPLAN +PLAN-ROOT SINK +| +06:AGGREGATE [FINALIZE] +| output: count:merge(*) +| row-size=8B cardinality=1 +| +05:EXCHANGE [UNPARTITIONED] +| +03:AGGREGATE +| output: count(*) +| row-size=8B cardinality=1 +| +02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED] +| row-size=20B cardinality=2 +| +|--04:EXCHANGE [DIRECTED] +| | +| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] +| HDFS partitions=1/1 files=1 size=2.63KB +| Iceberg snapshot id: 752781918366351945 +| row-size=267B cardinality=1 +| +00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] + HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 752781918366351945 row-size=20B cardinality=3 ==== SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files; @@ -374,10 +458,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -399,10 +485,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ==== SELECT * from iceberg_v2_positional_update_all_rows @@ -418,14 +506,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | HDFS partitions=1/1 files=1 size=2.60KB +| | Iceberg snapshot id: 3877007445826010687 | | row-size=246B cardinality=3 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 3877007445826010687 row-size=36B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -443,14 +534,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | HDFS partitions=1/1 files=1 size=2.60KB +| | Iceberg snapshot id: 3877007445826010687 | | row-size=246B cardinality=3 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] HDFS partitions=1/1 files=1 size=625B + Iceberg snapshot id: 3877007445826010687 row-size=36B cardinality=3 ==== SELECT * from iceberg_v2_partitioned_position_deletes @@ -462,10 +556,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=64B cardinality=20 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -479,10 +575,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=64B cardinality=20 ==== SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files @@ -499,16 +597,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i > 2 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i > 2 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -526,16 +627,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i > 2 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i > 2 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ==== select * from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 1497619269847778439 @@ -550,6 +654,7 @@ PLAN-ROOT SINK | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 4363979609026842966 | row-size=16B cardinality=3 | 05:AGGREGATE [FINALIZE] @@ -565,14 +670,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -587,6 +695,7 @@ PLAN-ROOT SINK | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 4363979609026842966 | row-size=16B cardinality=3 | 10:AGGREGATE [FINALIZE] @@ -610,14 +719,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== with v as (select i + 1000 as ii, upper(s) as ss from iceberg_v2_positional_not_all_data_files_have_delete_files) @@ -634,16 +746,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i + 1000 > 1003 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i + 1000 > 1003 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -661,16 +776,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | predicates: i + 1000 > 1003 +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB predicates: i + 1000 > 1003 + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=1 ==== select * @@ -697,14 +815,17 @@ PLAN-ROOT SINK | | | | | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=24B cardinality=3 | | | 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=24B cardinality=3 | 04:UNION @@ -716,16 +837,19 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | runtime filters: RF000 -> i +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB runtime filters: RF000 -> i + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -760,14 +884,17 @@ PLAN-ROOT SINK | | | | | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=24B cardinality=3 | | | 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=24B cardinality=3 | 04:UNION @@ -781,16 +908,19 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB | runtime filters: RF000 -> i +| Iceberg snapshot id: 1497619269847778439 | row-size=36B cardinality=6 | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB runtime filters: RF000 -> i + Iceberg snapshot id: 1497619269847778439 row-size=36B cardinality=4 ==== select * from iceberg_v2_partitioned_position_deletes where action = 'download' and id > 0; @@ -802,11 +932,13 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: id > 0 + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ---- DISTRIBUTEDPLAN @@ -821,11 +953,13 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: id > 0 + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ==== @@ -839,11 +973,13 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: `user` = 'Lisa' + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ---- DISTRIBUTEDPLAN @@ -858,11 +994,13 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB predicates: `user` = 'Lisa' + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=1 ==== @@ -873,6 +1011,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -883,6 +1022,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ==== @@ -893,6 +1033,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -903,6 +1044,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action IN ('click', 'view') row-size=32B cardinality=1 ==== @@ -913,6 +1055,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -923,6 +1066,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=6 size=6.85KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click') row-size=32B cardinality=1 ==== @@ -933,6 +1077,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view') row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -943,6 +1088,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view') row-size=32B cardinality=1 ==== @@ -953,6 +1099,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view')) row-size=32B cardinality=1 ---- DISTRIBUTEDPLAN @@ -963,6 +1110,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=14 size=15.93KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view')) row-size=32B cardinality=1 ==== @@ -973,6 +1121,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=20 size=22.90KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a') row-size=32B cardinality=2 ---- DISTRIBUTEDPLAN @@ -983,6 +1132,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=20 size=22.90KB predicates: id > 0 + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a') row-size=32B cardinality=2 ==== @@ -993,6 +1143,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=0 size=0B + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00' row-size=28B cardinality=0 ---- DISTRIBUTEDPLAN @@ -1002,6 +1153,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=0 size=0B + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00' row-size=28B cardinality=0 ==== @@ -1015,10 +1167,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=6 ---- DISTRIBUTEDPLAN @@ -1033,10 +1187,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.18KB +| Iceberg snapshot id: 8885697082976537578 | row-size=207B cardinality=2 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.17KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'download' row-size=64B cardinality=6 ==== @@ -1058,10 +1214,12 @@ PLAN-ROOT SINK | | | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=20B cardinality=6 | 09:NESTED LOOP JOIN [CROSS JOIN] @@ -1080,10 +1238,12 @@ PLAN-ROOT SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1110,10 +1270,12 @@ PLAN-ROOT SINK | | | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | row-size=267B cardinality=4 | | | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=2 size=1.22KB +| Iceberg snapshot id: 1497619269847778439 | row-size=20B cardinality=6 | 09:NESTED LOOP JOIN [CROSS JOIN, BROADCAST] @@ -1142,10 +1304,12 @@ PLAN-ROOT SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | HDFS partitions=1/1 files=2 size=5.33KB +| Iceberg snapshot id: 1497619269847778439 | row-size=267B cardinality=4 | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=2 size=1.22KB + Iceberg snapshot id: 1497619269847778439 row-size=20B cardinality=6 ==== select * from functional_parquet.iceberg_v2_delete_equality; @@ -1157,20 +1321,23 @@ PLAN-ROOT SINK | row-size=28B cardinality=3 | |--02:HASH JOIN [LEFT ANTI JOIN] -| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id +| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id | | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number | | row-size=28B cardinality=1 | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01] | | HDFS partitions=1/1 files=1 size=457B +| | Iceberg snapshot id: 5763349507283783091 | | row-size=16B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] | HDFS partitions=1/1 files=1 size=709B +| Iceberg snapshot id: 5763349507283783091 | row-size=28B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] HDFS partitions=1/1 files=2 size=1.41KB + Iceberg snapshot id: 5763349507283783091 row-size=28B cardinality=2 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1190,14 +1357,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01] | | HDFS partitions=1/1 files=1 size=457B +| | Iceberg snapshot id: 5763349507283783091 | | row-size=16B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] | HDFS partitions=1/1 files=1 size=709B +| Iceberg snapshot id: 5763349507283783091 | row-size=28B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] HDFS partitions=1/1 files=2 size=1.41KB + Iceberg snapshot id: 5763349507283783091 row-size=28B cardinality=2 ==== select * from functional_parquet.iceberg_v2_delete_equality for system_version as of 5763349507283783091; @@ -1215,14 +1385,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01] | | HDFS partitions=1/1 files=1 size=457B +| | Iceberg snapshot id: 5763349507283783091 | | row-size=16B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] | HDFS partitions=1/1 files=1 size=709B +| Iceberg snapshot id: 5763349507283783091 | row-size=28B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] HDFS partitions=1/1 files=2 size=1.41KB + Iceberg snapshot id: 5763349507283783091 row-size=28B cardinality=2 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1242,14 +1415,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01] | | HDFS partitions=1/1 files=1 size=457B +| | Iceberg snapshot id: 5763349507283783091 | | row-size=16B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] | HDFS partitions=1/1 files=1 size=709B +| Iceberg snapshot id: 5763349507283783091 | row-size=28B cardinality=1 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality] HDFS partitions=1/1 files=2 size=1.41KB + Iceberg snapshot id: 5763349507283783091 row-size=28B cardinality=2 ==== # Table has both equality and positional delete files. Equality fields are (i int, d date). @@ -1268,6 +1444,7 @@ PLAN-ROOT SINK | | | |--03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03] | | HDFS partitions=1/1 files=1 size=657B +| | Iceberg snapshot id: 911559291487642581 | | row-size=16B cardinality=2 | | | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] @@ -1275,14 +1452,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete] | | HDFS partitions=1/1 files=1 size=1.57KB +| | Iceberg snapshot id: 911559291487642581 | | row-size=187B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos] | HDFS partitions=1/1 files=1 size=898B +| Iceberg snapshot id: 911559291487642581 | row-size=48B cardinality=2 | 05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos] HDFS partitions=1/1 files=1 size=885B + Iceberg snapshot id: 911559291487642581 row-size=48B cardinality=2 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1302,6 +1482,7 @@ PLAN-ROOT SINK | | | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03] | | HDFS partitions=1/1 files=1 size=657B +| | Iceberg snapshot id: 911559291487642581 | | row-size=16B cardinality=2 | | | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED] @@ -1311,14 +1492,17 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete] | | HDFS partitions=1/1 files=1 size=1.57KB +| | Iceberg snapshot id: 911559291487642581 | | row-size=187B cardinality=1 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos] | HDFS partitions=1/1 files=1 size=898B +| Iceberg snapshot id: 911559291487642581 | row-size=48B cardinality=2 | 05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos] HDFS partitions=1/1 files=1 size=885B + Iceberg snapshot id: 911559291487642581 row-size=48B cardinality=2 ==== # Partitioned table with equality deletes. @@ -1337,14 +1521,17 @@ PLAN-ROOT SINK | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01] | | HDFS partitions=1/1 files=2 size=1.33KB +| | Iceberg snapshot id: 4821964189199835313 | | row-size=24B cardinality=3 | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned] | HDFS partitions=1/1 files=2 size=1.76KB +| Iceberg snapshot id: 4821964189199835313 | row-size=28B cardinality=5 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned] HDFS partitions=1/1 files=2 size=1.71KB + Iceberg snapshot id: 4821964189199835313 row-size=28B cardinality=3 ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1364,15 +1551,18 @@ PLAN-ROOT SINK | | | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01] | | HDFS partitions=1/1 files=2 size=1.33KB +| | Iceberg snapshot id: 4821964189199835313 | | row-size=24B cardinality=3 | | | 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_delete_equality_partitioned.d,functional_parquet.iceberg_v2_delete_equality_partitioned.s)] | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned] | HDFS partitions=1/1 files=2 size=1.76KB +| Iceberg snapshot id: 4821964189199835313 | row-size=28B cardinality=5 | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned] HDFS partitions=1/1 files=2 size=1.71KB + Iceberg snapshot id: 4821964189199835313 row-size=28B cardinality=3 ==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test index a7b61a2a6..57c27f3ef 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test @@ -7,6 +7,7 @@ MULTI DATA SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i = 3 + Iceberg snapshot id: 728158873687794725 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -16,6 +17,7 @@ MULTI DATA SINK 00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes] HDFS partitions=1/1 files=1 size=625B predicates: i = 3 + Iceberg snapshot id: 728158873687794725 row-size=36B cardinality=1 ==== UPDATE iceberg_v2_delete_positional SET `data` = concat(`data`,'a') where id = 15 @@ -29,11 +31,13 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: id = 15 + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=1 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -47,11 +51,13 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: id = 15 + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=1 ==== UPDATE iceberg_v2_delete_positional SET id = cast(id+1 as int) @@ -65,10 +71,12 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -82,10 +90,12 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=3 ==== UPDATE iceberg_v2_delete_positional SET id = 42 WHERE FILE__POSITION = id @@ -99,11 +109,13 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: FILE__POSITION = id + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=1 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -117,11 +129,13 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete] | HDFS partitions=1/1 files=1 size=1.54KB +| Iceberg snapshot id: 5725822353600261755 | row-size=182B cardinality=1 | 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional] HDFS partitions=1/1 files=1 size=662B predicates: FILE__POSITION = id + Iceberg snapshot id: 5725822353600261755 row-size=40B cardinality=1 ==== UPDATE iceberg_v2_partitioned_position_deletes set id = length(action) @@ -139,10 +153,12 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=76B cardinality=20 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -162,10 +178,12 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=3 size=3.48KB + Iceberg snapshot id: 8885697082976537578 row-size=76B cardinality=20 ==== UPDATE iceberg_v2_partitioned_position_deletes set id = length(action) where action = 'click' @@ -183,10 +201,12 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.15KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.15KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'click' row-size=76B cardinality=6 ---- DISTRIBUTEDPLAN @@ -205,10 +225,12 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete] | HDFS partitions=1/1 files=1 size=3.15KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=3 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes] HDFS partitions=1/1 files=1 size=1.15KB + Iceberg snapshot id: 8885697082976537578 skipped Iceberg predicates: action = 'click' row-size=76B cardinality=6 ==== @@ -236,14 +258,17 @@ MULTI DATA SINK | | | | | |--04:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-04 source-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=36B cardinality=3 | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] @@ -251,11 +276,13 @@ MULTI DATA SINK | |--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 target-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes target] HDFS partitions=1/1 files=3 size=3.48KB runtime filters: RF000 -> target.id + Iceberg snapshot id: 8885697082976537578 row-size=68B cardinality=20 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -286,14 +313,17 @@ MULTI DATA SINK | | | | | | | 04:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-04 source-position-delete] | | | HDFS partitions=1/1 files=1 size=2.60KB +| | | Iceberg snapshot id: 3877007445826010687 | | | row-size=246B cardinality=3 | | | | | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source] | | HDFS partitions=1/1 files=1 size=625B +| | Iceberg snapshot id: 3877007445826010687 | | row-size=36B cardinality=3 | | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 3877007445826010687 | row-size=36B cardinality=3 | 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED] @@ -303,11 +333,13 @@ MULTI DATA SINK | | | 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 target-position-delete] | HDFS partitions=1/1 files=3 size=9.47KB +| Iceberg snapshot id: 8885697082976537578 | row-size=204B cardinality=10 | 00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes target] HDFS partitions=1/1 files=3 size=3.48KB runtime filters: RF000 -> target.id + Iceberg snapshot id: 8885697082976537578 row-size=68B cardinality=20 ==== update iceberg_partition_transforms_zorder set ts = days_add(ts, 10), i = cast(i + 1000 as int) @@ -322,6 +354,7 @@ MULTI DATA SINK | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder] HDFS partitions=1/1 files=1 size=1.08KB + Iceberg snapshot id: 7350750578864730166 row-size=72B cardinality=1 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -334,6 +367,7 @@ MULTI DATA SINK | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder] HDFS partitions=1/1 files=1 size=1.08KB + Iceberg snapshot id: 7350750578864730166 row-size=72B cardinality=1 ==== update ice_zorder set j = length(action) @@ -355,11 +389,13 @@ MULTI DATA SINK | |--00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder] | HDFS partitions=1/1 files=1 size=1.08KB +| Iceberg snapshot id: 7350750578864730166 | row-size=68B cardinality=1 | 01:SCAN HDFS [functional_parquet.iceberg_partitioned source] HDFS partitions=1/1 files=20 size=22.90KB runtime filters: RF000 -> source.id + Iceberg snapshot id: 8270633197658268308 row-size=16B cardinality=20 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -381,11 +417,13 @@ MULTI DATA SINK | | | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder] | HDFS partitions=1/1 files=1 size=1.08KB +| Iceberg snapshot id: 7350750578864730166 | row-size=68B cardinality=1 | 01:SCAN HDFS [functional_parquet.iceberg_partitioned source] HDFS partitions=1/1 files=20 size=22.90KB runtime filters: RF000 -> source.id + Iceberg snapshot id: 8270633197658268308 row-size=16B cardinality=20 ==== update ice_zorder set j = length(action) @@ -407,11 +445,13 @@ MULTI DATA SINK | |--00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder] | HDFS partitions=1/1 files=1 size=1.08KB +| Iceberg snapshot id: 7350750578864730166 | row-size=68B cardinality=1 | 01:SCAN HDFS [functional_parquet.iceberg_partitioned source] HDFS partitions=1/1 files=20 size=22.90KB runtime filters: RF000 -> source.id + Iceberg snapshot id: 8270633197658268308 row-size=16B cardinality=20 ---- DISTRIBUTEDPLAN MULTI DATA SINK @@ -433,10 +473,12 @@ MULTI DATA SINK | | | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder] | HDFS partitions=1/1 files=1 size=1.08KB +| Iceberg snapshot id: 7350750578864730166 | row-size=68B cardinality=1 | 01:SCAN HDFS [functional_parquet.iceberg_partitioned source] HDFS partitions=1/1 files=20 size=22.90KB runtime filters: RF000 -> source.id + Iceberg snapshot id: 8270633197658268308 row-size=16B cardinality=20 ==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test b/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test index 45c5d186e..7aa8965e3 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test @@ -401,6 +401,7 @@ WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder] HDFS partitions=1/1 files=1 size=1.08KB + Iceberg snapshot id: 7350750578864730166 row-size=36B cardinality=1 ---- DISTRIBUTEDPLAN WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE=true, PARTITION-KEYS=(year(functional_parquet.iceberg_partition_transforms_zorder.ts),iceberg_bucket_transform(functional_parquet.iceberg_partition_transforms_zorder.s, 5))] @@ -413,5 +414,6 @@ WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE | 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder] HDFS partitions=1/1 files=1 size=1.08KB + Iceberg snapshot id: 7350750578864730166 row-size=36B cardinality=1 ==== diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test index 795e7ff79..6c3b986f5 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test @@ -258,6 +258,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_non_partitioned] HDFS partitions=1/1 files=3 size=3.41KB + Iceberg snapshot id: 93996984692289973 stored statistics: table: rows=20 size=22.90KB columns: unavailable @@ -283,6 +284,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_non_partitioned] HDFS partitions=1/1 files=3 size=3.41KB + Iceberg snapshot id: 93996984692289973 stored statistics: table: rows=20 size=22.90KB columns: all @@ -302,6 +304,7 @@ PLAN-ROOT SINK | 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=10 size=11.46KB + Iceberg snapshot id: 8270633197658268308 stored statistics: table: rows=20 size=22.90KB columns: unavailable @@ -324,6 +327,7 @@ PLAN-ROOT SINK 00:SCAN HDFS [functional_parquet.iceberg_partitioned] HDFS partitions=1/1 files=4 size=4.57KB predicates: id > CAST(0 AS INT) + Iceberg snapshot id: 8270633197658268308 skipped Iceberg predicates: action = 'click' stored statistics: table: rows=20 size=22.90KB @@ -349,17 +353,18 @@ PLAN-ROOT SINK 04:UNION | pass-through-operands: all | mem-estimate=0B mem-reservation=0B thread-reservation=0 -| tuple-ids=0 row-size=36B cardinality=4 +| tuple-ids=0 row-size=36B cardinality=2 | in pipelines: 03(GETNEXT), 00(GETNEXT) | |--02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN] | | equality predicates: functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position = functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete.pos, functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name = functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete.file_path | | mem-estimate=566B mem-reservation=566B thread-reservation=0 -| | tuple-ids=0 row-size=36B cardinality=3 +| | tuple-ids=0 row-size=36B cardinality=1 | | in pipelines: 00(GETNEXT), 01(OPEN) | | | |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete] | | HDFS partitions=1/1 files=2 size=5.33KB +| | Iceberg snapshot id: 1497619269847778439 | | stored statistics: | | table: rows=4 size=5.33KB | | columns: all @@ -370,6 +375,7 @@ PLAN-ROOT SINK | | | 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] | HDFS partitions=1/1 files=1 size=625B +| Iceberg snapshot id: 1497619269847778439 | stored statistics: | table: rows=10 size=7.77KB | columns missing stats: i, s @@ -380,6 +386,7 @@ PLAN-ROOT SINK | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files] HDFS partitions=1/1 files=1 size=620B + Iceberg snapshot id: 1497619269847778439 stored statistics: table: rows=10 size=7.77KB columns missing stats: i, s diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py index 8679a3e91..ed34262a6 100644 --- a/tests/query_test/test_iceberg.py +++ b/tests/query_test/test_iceberg.py @@ -462,6 +462,12 @@ class TestIcebergTable(IcebergTestSuite): "select count(*) from {0} for system_time as of {1}".format(tbl_name, ts), expected) + def expect_snapshot_id_in_plan_t(ts, snapshot_id): + data = impalad_client.execute( + "explain select * from {0} for system_time as of {1}".format( + tbl_name, ts)) + assert " Iceberg snapshot id: {0}".format(snapshot_id) in data.data + def expect_results_v(snapshot_id, expected_results, expected_cols): expect_results( "select * from {0} for system_version as of {1}".format(tbl_name, snapshot_id), @@ -473,6 +479,13 @@ class TestIcebergTable(IcebergTestSuite): tbl_name, snapshot_id), expected) + def expect_snapshot_id_in_plan_v(snapshot_id): + data = impalad_client.execute( + "explain select * from {0} for system_version as of {1}".format( + tbl_name, snapshot_id)) + assert " Iceberg snapshot id: {0}".format(snapshot_id) in data.data + + def impala_now(): now_data = impalad_client.execute("select now()") return now_data.data[0] @@ -540,6 +553,18 @@ class TestIcebergTable(IcebergTestSuite): expect_results_v(snapshots[3].get_snapshot_id(), ['100'], i_cols) expect_results_v(snapshots[4].get_snapshot_id(), ['100\tNULL', '3\t103'], ij_cols) + expect_snapshot_id_in_plan_v(snapshots[0].get_snapshot_id()) + expect_snapshot_id_in_plan_v(snapshots[1].get_snapshot_id()) + expect_snapshot_id_in_plan_v(snapshots[2].get_snapshot_id()) + expect_snapshot_id_in_plan_v(snapshots[3].get_snapshot_id()) + expect_snapshot_id_in_plan_v(snapshots[4].get_snapshot_id()) + + expect_snapshot_id_in_plan_t(quote(ts_1), snapshots[0].get_snapshot_id()) + expect_snapshot_id_in_plan_t(quote(ts_2), snapshots[1].get_snapshot_id()) + expect_snapshot_id_in_plan_t(quote(ts_3), snapshots[2].get_snapshot_id()) + expect_snapshot_id_in_plan_t(quote(ts_4), snapshots[3].get_snapshot_id()) + expect_snapshot_id_in_plan_t(quote(ts_5), snapshots[4].get_snapshot_id()) + # Test of plain count star optimization # 'NumRowGroups' and 'NumFileMetadataRead' should not appear in profile expect_for_count_star_t("now()", '2')
