This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 843ecdf368b [fix](iceberg) Fix s3tables system table (#53137)
843ecdf368b is described below
commit 843ecdf368bb5e12a0b5cb86189acf40ed47a7b0
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sat Jul 12 12:29:23 2025 -0700
[fix](iceberg) Fix s3tables system table (#53137)
### What problem does this PR solve?
Problem Summary:
We miss some dependency on BE side for s3 tables, followup #51190
---
.../iceberg-metadata-scanner/pom.xml | 6 +++
.../s3tables/CustomAwsCredentialsProvider.java | 45 +++++++++++++++++++++
fe/be-java-extensions/preload-extensions/pom.xml | 16 ++++++++
.../iceberg/test_iceberg_equality_delete.out | Bin 407 -> 477 bytes
.../iceberg/test_s3tables_write_partitions.out | Bin 267 -> 291 bytes
.../iceberg/iceberg_query_tag_branch.groovy | 29 +++++++++++++
.../iceberg/test_iceberg_equality_delete.groovy | 21 +++++++++-
.../iceberg/test_iceberg_position_delete.groovy | 2 +-
.../iceberg/test_s3tables_write_partitions.groovy | 19 +++++++++
9 files changed, 136 insertions(+), 2 deletions(-)
diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml
b/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml
index f563ddf1837..d3c4141a786 100644
--- a/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml
+++ b/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml
@@ -52,6 +52,12 @@ under the License.
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>auth</artifactId>
+ <version>${awssdk.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
new file mode 100644
index 00000000000..4f35ac930da
--- /dev/null
+++
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.s3tables;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import java.util.Map;
+
+// Copy from
+//
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/s3tables/CustomAwsCredentialsProvider.java
+public class CustomAwsCredentialsProvider implements AwsCredentialsProvider {
+ private final String accessKeyId;
+ private final String secretAccessKey;
+
+ public CustomAwsCredentialsProvider(String accessKeyId, String
secretAccessKey) {
+ this.accessKeyId = accessKeyId;
+ this.secretAccessKey = secretAccessKey;
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return AwsBasicCredentials.create(accessKeyId, secretAccessKey);
+ }
+
+ public static CustomAwsCredentialsProvider create(Map<String, String>
props) {
+ return new CustomAwsCredentialsProvider(props.get("s3.access-key-id"),
props.get("s3.secret-access-key"));
+ }
+}
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml
b/fe/be-java-extensions/preload-extensions/pom.xml
index b64f450de18..077459d816f 100644
--- a/fe/be-java-extensions/preload-extensions/pom.xml
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -242,6 +242,22 @@ under the License.
<artifactId>dynamodb</artifactId>
<version>${awssdk.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3tables</artifactId>
+ <version>${awssdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.s3tables</groupId>
+ <artifactId>s3-tables-catalog-for-iceberg</artifactId>
+ <version>${s3tables.catalog.version}</version>
+ </dependency>
+ <!-- AWS SDK Core -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sdk-core</artifactId>
+ <version>${awssdk.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
index 3a21417f1a7..5bb2deb36d9 100644
Binary files
a/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
and
b/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
differ
diff --git
a/regression-test/data/external_table_p2/iceberg/test_s3tables_write_partitions.out
b/regression-test/data/external_table_p2/iceberg/test_s3tables_write_partitions.out
index b206cd3d90d..c0b4eb6ece7 100644
Binary files
a/regression-test/data/external_table_p2/iceberg/test_s3tables_write_partitions.out
and
b/regression-test/data/external_table_p2/iceberg/test_s3tables_write_partitions.out
differ
diff --git
a/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy
b/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy
index 6ada85953da..4062a6df649 100644
---
a/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy
@@ -235,6 +235,35 @@ suite("iceberg_query_tag_branch",
"p0,external,doris,external_docker,external_do
query_tag_branch_in_subquery()
query_exception()
+ // test sql. no result check because snapshot_id is changed every time
+ sql """
+ SELECT
+ refs_data.snapshot_id,
+ snapshots.committed_at,
+ snapshots.operation,
+ ARRAY_SORT(refs_data.refs)
+ FROM (
+ SELECT
+ snapshot_id,
+ ARRAY_AGG(CONCAT(type, ':', name)) AS refs
+ FROM
+ tag_branch_table\$refs
+ GROUP BY
+ snapshot_id
+ ) AS refs_data
+ JOIN (
+ SELECT
+ snapshot_id,
+ committed_at,
+ operation
+ FROM
+ tag_branch_table\$snapshots
+ ) AS snapshots
+ ON refs_data.snapshot_id = snapshots.snapshot_id
+ ORDER BY
+ snapshots.committed_at;
+ """
+
} finally {
// Restore default values
sql """ set num_files_in_batch_mode=1024; """
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
index 74a3dc953eb..78b98f24b2f 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
@@ -60,7 +60,26 @@ suite("test_iceberg_equality_delete",
"p0,external,doris,external_docker,externa
qt_max3 """select max(val) from customer_flink_three"""
qt_max3_orc """select max(val) from customer_flink_three_orc"""
- sql """drop catalog ${catalog_name}"""
+ // test sql
+ qt_test_sql """
+ SELECT
+ CASE
+ WHEN content = 0 THEN 'DataFile'
+ WHEN content = 1 THEN 'PositionDeleteFile'
+ WHEN content = 2 THEN 'EqualityDeleteFile'
+ ELSE 'Unknown'
+ END AS ContentType,
+ -- SUM(file_size_in_bytes) AS SizeInBytes,
+ SUM(record_count) AS Records
+ FROM
+ test_db.customer_flink_three_orc\$files
+ GROUP BY
+ ContentType
+ ORDER BY
+ ContentType;
+ """
+
+ // sql """drop catalog ${catalog_name}"""
}
/*
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
index 6a14712a2af..d793cef3568 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
@@ -164,7 +164,7 @@ suite("test_iceberg_position_delete",
"p0,external,doris,external_docker,externa
List<List<Object>> iceberg_position_gen_7 = sql """select id from
iceberg_position_gen_data where name != "hello word" ;"""
assertTrue(iceberg_position_gen_7.size() == 5632)
- sql """drop catalog ${catalog_name}"""
+ // sql """drop catalog ${catalog_name}"""
}
/*
diff --git
a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy
b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy
index a3b23981ab9..d322cd5ab03 100644
---
a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy
+++
b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy
@@ -105,4 +105,23 @@ suite("test_s3tables_write_partitions",
"p0,external,iceberg,external_docker,ext
} finally {
}
+ //test sql
+ sql """ switch ${catalog_name};"""
+ sql """ use my_namespace;"""
+ order_qt_test_sql """
+ SELECT
+ CASE
+ WHEN file_size_in_bytes BETWEEN 0 AND 8 * 1024 * 1024 THEN '0-8M'
+ WHEN file_size_in_bytes BETWEEN 8 * 1024 * 1024 + 1 AND 32 * 1024
* 1024 THEN '8-32M'
+ WHEN file_size_in_bytes BETWEEN 2 * 1024 * 1024 + 1 AND 128 * 1024
* 1024 THEN '32-128M'
+ WHEN file_size_in_bytes BETWEEN 128 * 1024 * 1024 + 1 AND 512 *
1024 * 1024 THEN '128-512M'
+ WHEN file_size_in_bytes > 512 * 1024 * 1024 THEN '> 512M'
+ ELSE 'Unknown'
+ END AS SizeRange,
+ COUNT(*) AS FileNum
+ FROM partitioned_table\$data_files
+ GROUP BY
+ SizeRange;
+ """
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
