This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 12242f201dd branch-3.1: [opt](show-partitions) remove show partitions
support for iceberg (#56985) (#57501)
12242f201dd is described below
commit 12242f201dde5c6f20e5528cfe996cf18a2ce8c6
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Fri Oct 31 17:53:34 2025 +0800
branch-3.1: [opt](show-partitions) remove show partitions support for
iceberg (#56985) (#57501)
bp #56985
---
.../datasource/iceberg/IcebergExternalTable.java | 14 +--
.../maxcompute/MaxComputeExternalTable.java | 6 ++
.../datasource/paimon/PaimonExternalTable.java | 6 ++
regression-test/data/mtmv_p0/test_iceberg_mtmv.out | 65 ++++++++----
.../suites/mtmv_p0/test_iceberg_mtmv.groovy | 109 +++++++++++----------
5 files changed, 124 insertions(+), 76 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
index 1a4ccfe87bb..d4032a8628e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -66,7 +66,6 @@ import java.util.stream.Collectors;
public class IcebergExternalTable extends ExternalTable implements
MTMVRelatedTableIf, MTMVBaseTableIf, MvccTable {
- private Table table;
private boolean isValidRelatedTableCached = false;
private boolean isValidRelatedTable = false;
private boolean isView;
@@ -89,11 +88,6 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
}
}
- @VisibleForTesting
- public void setTable(Table table) {
- this.table = table;
- }
-
@Override
public Optional<SchemaCacheValue> initSchema(SchemaCacheKey key) {
boolean isView = isView();
@@ -209,7 +203,7 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
}
isValidRelatedTable = false;
Set<String> allFields = Sets.newHashSet();
- table = getIcebergTable();
+ Table table = getIcebergTable();
for (PartitionSpec spec : table.specs().values()) {
if (spec == null) {
isValidRelatedTableCached = true;
@@ -364,4 +358,10 @@ public class IcebergExternalTable extends ExternalTable
implements MTMVRelatedTa
}
}
+ @Override
+ public boolean isPartitionedTable() {
+ makeSureInitialized();
+ Table table = getIcebergTable();
+ return table.spec().isPartitioned();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index 7135ef96fdc..d9d167e786b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -345,4 +345,10 @@ public class MaxComputeExternalTable extends ExternalTable
{
return schemaCacheValue.map(value -> ((MaxComputeSchemaCacheValue)
value).getOdpsTable())
.orElse(null);
}
+
+ @Override
+ public boolean isPartitionedTable() {
+ makeSureInitialized();
+ return getOdpsTable().isPartitioned();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 06983786849..add9413b491 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -352,4 +352,10 @@ public class PaimonExternalTable extends ExternalTable
implements MTMVRelatedTab
return Collections.emptyMap();
}
}
+
+ @Override
+ public boolean isPartitionedTable() {
+ makeSureInitialized();
+ return !paimonTable.partitionKeys().isEmpty();
+ }
}
diff --git a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
index 47d2d3a61a2..4a3a169a022 100644
--- a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
@@ -59,10 +59,10 @@
\N 8
-- !test_iceberg_table_partition_ts --
-ts_day=20022 types: [DATETIMEV2]; keys: [2024-10-26 00:00:00]; types:
[DATETIMEV2]; keys: [2024-10-27 00:00:00];
-ts_day=20023 types: [DATETIMEV2]; keys: [2024-10-27 00:00:00]; types:
[DATETIMEV2]; keys: [2024-10-28 00:00:00];
-ts_day=20024 types: [DATETIMEV2]; keys: [2024-10-28 00:00:00]; types:
[DATETIMEV2]; keys: [2024-10-29 00:00:00];
-ts_day=null types: [DATETIMEV2]; keys: [0000-01-01 00:00:00]; types:
[DATETIMEV2]; keys: [0000-01-01 00:00:01];
+{"ts_day":null}
+{"ts_day":"2024-10-26"}
+{"ts_day":"2024-10-27"}
+{"ts_day":"2024-10-28"}
-- !unpartition --
2024-10-26T01:02:03 1
@@ -110,9 +110,9 @@ ts_day=null types: [DATETIMEV2]; keys: [0000-01-01
00:00:00]; types: [DATETIMEV
2024-10-28 7
-- !test_iceberg_table_partition_d --
-d_month=655 types: [DATEV2]; keys: [2024-08-01]; types: [DATEV2]; keys:
[2024-09-01];
-d_month=656 types: [DATEV2]; keys: [2024-09-01]; types: [DATEV2]; keys:
[2024-10-01];
-d_month=657 types: [DATEV2]; keys: [2024-10-01]; types: [DATEV2]; keys:
[2024-11-01];
+{"d_month":655}
+{"d_month":656}
+{"d_month":657}
-- !refresh_one_partition --
2024-01-01T00:00 4
@@ -130,21 +130,50 @@ d_month=657 types: [DATEV2]; keys: [2024-10-01];
types: [DATEV2]; keys: [2024-1
2024-01-02T00:00 3
-- !evolution2 --
-ts_day=20060 types: [DATETIMEV2]; keys: [2024-12-03 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-04 00:00:00];
-ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-11-01 00:00:00];
-ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-01 00:00:00];
+{"ts_month":null, "ts_day":"2024-12-03"}
+{"ts_month":657, "ts_day":null}
+{"ts_month":658, "ts_day":null}
-- !evolution3 --
-ts_day=20059 types: [DATETIMEV2]; keys: [2024-12-02 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-03 00:00:00];
-ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-01 00:00:00];
+{"ts_month":null, "ts_day":"2024-12-02"}
+{"ts_month":null, "ts_day":"2024-11-02"}
+{"ts_month":null, "ts_day":"2024-11-03"}
+{"ts_month":658, "ts_day":null}
-- !evolution4 --
-ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-11-01 00:00:00];
-ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-01 00:00:00];
+{"ts_month":null, "ts_day":"2024-11-02"}
+{"ts_month":null, "ts_day":"2024-11-03"}
+{"ts_month":657, "ts_day":null}
+{"ts_month":658, "ts_day":null}
-- !evolution5 --
-ts_day=20078 types: [DATETIMEV2]; keys: [2024-12-21 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-22 00:00:00];
-ts_hour=482139 types: [DATETIMEV2]; keys: [2025-01-01 03:00:00]; types:
[DATETIMEV2]; keys: [2025-01-01 04:00:00];
-ts_month=657 types: [DATETIMEV2]; keys: [2024-10-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-11-01 00:00:00];
-ts_month=658 types: [DATETIMEV2]; keys: [2024-11-01 00:00:00]; types:
[DATETIMEV2]; keys: [2024-12-01 00:00:00];
+{"ts_month":null, "ts_day":"2024-10-12", "ts_hour":null}
+{"ts_month":null, "ts_day":null, "ts_hour":482139}
+{"ts_month":null, "ts_day":"2024-12-21", "ts_hour":null}
+{"ts_month":null, "ts_day":null, "ts_hour":481875}
+{"ts_month":657, "ts_day":null, "ts_hour":null}
+{"ts_month":658, "ts_day":null, "ts_hour":null}
+
+-- !test1 --
+{"create_date1_month":null, "create_date2_year":53}
+{"create_date1_month":647, "create_date2_year":null}
+
+-- !test2 --
+{"create_date_bucket":2}
+{"create_date_bucket":5}
+
+-- !test3 --
+{"create_date_month":647}
+
+-- !test4 --
+{"create_date1_month":647, "create_date2_year":53}
+{"create_date1_month":647, "create_date2_year":null}
+
+-- !test5 --
+{"create_date_month":null}
+{"create_date_month":647}
+
+-- !test6 --
+{"create_date1_month":647, "create_date2_year":53}
+{"create_date1_month":647, "create_date2_year":null}
diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
index f0019f2cb8b..edbd9a34633 100644
--- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
@@ -137,7 +137,7 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh_null """select * from ${mvName1} order by value"""
- qt_test_iceberg_table_partition_ts """show partitions from
${catalog_name}.${icebergDb}.${icebergTable1};"""
+ qt_test_iceberg_table_partition_ts """select `partition` from
${catalog_name}.${icebergDb}.${icebergTable1}\$partitions order by
struct_element(`partition`, 1);"""
def showPartitionsResult = sql """show partitions from ${mvName1}"""
logger.info("showPartitionsResult: " + showPartitionsResult.toString())
@@ -198,7 +198,7 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh5 "select * from ${mvName2} order by value"
- qt_test_iceberg_table_partition_d """show partitions from
${catalog_name}.${icebergDb}.${icebergTable2};"""
+ qt_test_iceberg_table_partition_d """select `partition` from
${catalog_name}.${icebergDb}.${icebergTable2}\$partitions order by
struct_element(`partition`, 1);"""
showPartitionsResult = sql """show partitions from ${mvName2}"""
logger.info("showPartitionsResult: " + showPartitionsResult.toString())
@@ -241,7 +241,7 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
assertTrue(showPartitions.toString().contains("p_20240101000000_20240102000000"))
assertTrue(showPartitions.toString().contains("p_20240102000000_20240103000000"))
- // refresh one partiton
+ // refresh one partition
sql """REFRESH MATERIALIZED VIEW ${mvName}
partitions(p_20240101000000_20240102000000);"""
waitingMTMVTaskFinishedByMvName(mvName, dbName)
order_qt_refresh_one_partition "SELECT * FROM ${mvName} "
@@ -265,54 +265,61 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
sql """drop table if exists
${catalog_name}.${icebergDb}.${icebergTable3}"""
sql """use ${catalog_name}.test_db"""
- qt_evolution2 "show partitions from replace_partition2"
- qt_evolution3 "show partitions from replace_partition3"
- qt_evolution4 "show partitions from replace_partition4"
- qt_evolution5 "show partitions from replace_partition5"
-
- test {
- sql "show partitions from replace_partition1"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from no_partition"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from not_support_trans"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from drop_partition1"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from drop_partition2"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from add_partition1"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- test {
- sql "show partitions from add_partition2"
- // check exception message contains
- exception "is not a supported partition table"
- }
-
- sql """ drop catalog if exists ${catalog_name} """
+ qt_evolution2 "select `partition` from replace_partition2\$partitions
order by struct_element(`partition`, 1)"
+ qt_evolution3 "select `partition` from replace_partition3\$partitions
order by struct_element(`partition`, 1)"
+ qt_evolution4 "select `partition` from replace_partition4\$partitions
order by struct_element(`partition`, 1)"
+ qt_evolution5 "select `partition` from replace_partition5\$partitions
order by struct_element(`partition`, 1)"
+
+ qt_test1 "select `partition` from replace_partition1\$partitions order
by struct_element(`partition`, 1)";
+ qt_test2 "select `partition` from not_support_trans\$partitions order
by struct_element(`partition`, 1)";
+ qt_test3 "select `partition` from drop_partition1\$partitions order by
struct_element(`partition`, 1)";
+ qt_test4 "select `partition` from drop_partition2\$partitions order by
struct_element(`partition`, 1)";
+ qt_test5 "select `partition` from add_partition1\$partitions order by
struct_element(`partition`, 1)";
+ qt_test6 "select `partition` from add_partition2\$partitions order by
struct_element(`partition`, 1)";
+
+ // test {
+ // sql "show partitions from replace_partition1"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // test {
+ // sql "show partitions from no_partition"
+ // // check exception message contains
+ // exception "Unknown column 'partition'"
+ // }
+
+ // test {
+ // sql "show partitions from not_support_trans"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // test {
+ // sql "show partitions from drop_partition1"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // test {
+ // sql "show partitions from drop_partition2"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // test {
+ // sql "show partitions from add_partition1"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // test {
+ // sql "show partitions from add_partition2"
+ // // check exception message contains
+ // exception "is not a supported partition table"
+ // }
+
+ // sql """ drop catalog if exists ${catalog_name} """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
