This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 0eef45d593a branch-4.1: [fix](test) Fix external_table_p0 DATETIME
baselines and Iceberg object counting #66970 (#66994)
0eef45d593a is described below
commit 0eef45d593a35b9c7df20d5a0be70d0157f96f1e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Aug 21 10:11:36 2026 +0800
branch-4.1: [fix](test) Fix external_table_p0 DATETIME baselines and
Iceberg object counting #66970 (#66994)
Cherry-picked from #66970
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../test_remote_doris_all_types_select.out | 4 +-
.../remote_doris/test_remote_doris_statistics.out | 2 +-
...rg_write_merge_duplicate_source_negative.groovy | 73 +++++++++++++++-------
3 files changed, 54 insertions(+), 25 deletions(-)
diff --git
a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
index faa3cb627ab..a8823bb027e 100644
---
a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
+++
b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
@@ -1,12 +1,12 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
-2025-05-18T01:00 true -128 -32768 -2147483648
-9223372036854775808 -1234567890123456790 -123.456 -123456.789
-123457 -123456789012346 -1234567890123456789012345678 1970-01-01
A Hello Hello, Doris! ["apple", "banana", "orange"]
{"Emily":101, "age":25} {"f1":11, "f2":3.14, "f3":"Emily"}
{"k1":"v31","k2":300,"k3":[123,456],"k4":[],"k5":{"i1":"iv1"}}
+2025-05-18T01:00 true -128 -32768 -2147483648
-9223372036854775808 -1234567890123456790 -123.456 -123456.789
-123457 -123456789012346 -1234567890123456789012345678 1970-01-01
0000-01-01T00:00 A Hello Hello, Doris! ["apple", "banana",
"orange"] {"Emily":101, "age":25} {"f1":11, "f2":3.14, "f3":"Emily"}
{"k1":"v31","k2":300,"k3":[123,456],"k4":[],"k5":{"i1":"iv1"}}
2025-05-18T02:00 \N \N \N \N \N \N \N
\N \N \N \N \N \N \N \N \N \N
\N \N \N
2025-05-18T03:00 false 127 32767 2147483647
9223372036854775807 1234567890123456789 123.456 123456.789 123457
123456789012346 1234567890123456789012345678 9999-12-31
9999-12-31T23:59:59 [] {} {"f1":11,
"f2":3.14, "f3":"Emily"} {}
2025-05-18T04:00 true 0 0 0 0 0 0.0
0 0 0 0 2023-10-01 2023-10-01T12:34:56 A
Hello Hello, Doris! ["apple", "banana", "orange"] {"Emily":101, "age":25}
{"f1":11, "f2":3.14, "f3":"Emily"} []
-- !sql --
-2025-05-18T01:00 [1] [-128] [-32768] [-2147483648]
[-9223372036854775808] [-1234567890123456790] [-123.456] [-123456.789]
[-123457] [-123456789012346] [-1234567890123456789012345678]
["0000-01-01"] [""] ["A"] ["Hello"] ["Hello, Doris!"]
+2025-05-18T01:00 [1] [-128] [-32768] [-2147483648]
[-9223372036854775808] [-1234567890123456790] [-123.456] [-123456.789]
[-123457] [-123456789012346] [-1234567890123456789012345678]
["0000-01-01"] ["0000-01-01 00:00:00"] ["A"] ["Hello"] ["Hello,
Doris!"]
2025-05-18T02:00 [null] [null] [null] [null] [null] [null] [null]
[null] [null] [null] [null] [null] [null] [null] [null] [null]
2025-05-18T03:00 [0] [127] [32767] [2147483647]
[9223372036854775807] [1234567890123456789] [123.456] [123456.789]
[123457] [123456789012346] [1234567890123456789012345678]
["9999-12-31"] ["9999-12-31 23:59:59"] [""] [""] [""]
2025-05-18T04:00 [1] [0] [0] [0] [0] [0] [0]
[0] [0] [0] [0] ["2023-10-01"] ["2023-10-01 12:34:56"] ["A"]
["Hello"] ["Hello, Doris!"]
diff --git
a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
index 1d915ff3620..eecba5cbcd6 100644
---
a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
+++
b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
@@ -4,7 +4,7 @@ c_bigint 4 3 1 -9223372036854775808
9223372036854775807 32
c_boolean 4 2 1 0 1 4
c_char 4 2 1 A 2
c_date 4 3 1 1970-01-01 9999-12-31 16
-c_datetime 4 3 1 2023-10-01 12:34:56 32
+c_datetime 4 3 1 0000-01-01 00:00:00 9999-12-31
23:59:59 32
c_decimal18 4 3 1 -123456789012346 123456789012346
32
c_decimal32 4 3 1 -1234567890123456789012345678
1234567890123456789012345678 64
c_decimal9 4 3 1 -123457 123457 16
diff --git
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy
index 075bfccad96..17ffc29d0cd 100644
---
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy
@@ -15,6 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+import com.amazonaws.auth.AWSStaticCredentialsProvider
+import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
+import com.amazonaws.services.s3.AmazonS3ClientBuilder
+import com.amazonaws.services.s3.model.ListObjectsV2Request
+
suite("test_iceberg_write_merge_duplicate_source_negative",
"p0,external,iceberg,external_docker,external_docker_iceberg,nonConcurrent") {
String enabled = context.config.otherConfigs.get("enableIcebergTest")
@@ -27,21 +33,36 @@ suite("test_iceberg_write_merge_duplicate_source_negative",
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String catalogName = "test_iceberg_write_merge_duplicate_source_negative"
String dbName = "iceberg_write_merge_duplicate_source_negative_db"
- String dockerCommand =
context.config.otherConfigs.get("externalDockerCommand") ?: "docker"
- String mcImage = "minio/mc:RELEASE.2025-01-17T23-25-50Z"
- def countDataObjects = { String objectPath ->
- def stdout = new StringBuilder()
- def stderr = new StringBuilder()
- def process = new ProcessBuilder("/bin/bash", "-c",
- "${dockerCommand} run --rm --entrypoint /bin/sh ${mcImage} -c "
- + "'/usr/bin/mc alias set minio
http://${externalEnvIp}:${minioPort} "
- + "admin password >/dev/null "
- + "&& /usr/bin/mc find ${objectPath} --type f | wc
-l'").start()
- process.consumeProcessOutput(stdout, stderr)
- process.waitForOrKill(30000)
- assertEquals(0, process.exitValue(), "Failed to list Iceberg data
objects: ${stderr}")
- return stdout.toString().trim() as long
+ // MinIO only publishes its API on the host, while its container sits on
the isolated
+ // `doris--iceberg` compose network. Listing objects from this JVM reuses
the very endpoint the
+ // catalog is configured with. Shelling out to a throwaway `minio/mc`
container instead puts the
+ // client on the default bridge, so every call has to hairpin back to the
published port, which
+ // intermittently times out while the agent starts and stops other
containers.
+ def minioClient = AmazonS3ClientBuilder.standard()
+ .withEndpointConfiguration(
+ new
EndpointConfiguration("http://${externalEnvIp}:${minioPort}";, "us-east-1"))
+ .withPathStyleAccessEnabled(true)
+ .withCredentials(new AWSStaticCredentialsProvider(
+ new BasicAWSCredentials("admin", "password")))
+ .build()
+
+ def countDataObjects = { String bucket, String prefix ->
+ long objectCount = 0
+ String continuationToken = null
+ while (true) {
+ def listing = minioClient.listObjectsV2(new ListObjectsV2Request()
+ .withBucketName(bucket)
+ .withPrefix(prefix)
+ .withContinuationToken(continuationToken))
+ // Directory placeholder keys are not data objects, mirroring `mc
find --type f`.
+ objectCount += listing.getObjectSummaries().count {
!it.getKey().endsWith("/") }
+ if (!listing.isTruncated()) {
+ break
+ }
+ continuationToken = listing.getNextContinuationToken()
+ }
+ return objectCount
}
sql """drop catalog if exists ${catalogName}"""
@@ -81,11 +102,18 @@ suite("test_iceberg_write_merge_duplicate_source_negative",
String committedFile = (sql """
select file_path from duplicate_source_target\$files order by
file_path limit 1
""")[0][0].toString()
- int dataDirectoryEnd = committedFile.indexOf('/data/') + '/data'.length()
- assertTrue(dataDirectoryEnd >= '/data'.length(), "Unexpected Iceberg data
path: ${committedFile}")
- String dataObjectPath = committedFile.substring(0, dataDirectoryEnd)
- .replaceFirst('^s3a?://warehouse', 'minio/warehouse')
- long objectsBefore = countDataObjects(dataObjectPath)
+ // file_path is `s3://<bucket>/<key>`; split it into the bucket and the
table's data prefix.
+ int schemeEnd = committedFile.indexOf("://")
+ assertTrue(schemeEnd > 0, "Unexpected Iceberg data path: ${committedFile}")
+ String bucketAndKey = committedFile.substring(schemeEnd + "://".length())
+ int bucketEnd = bucketAndKey.indexOf('/')
+ assertTrue(bucketEnd > 0, "Unexpected Iceberg data path: ${committedFile}")
+ String dataBucket = bucketAndKey.substring(0, bucketEnd)
+ String objectKey = bucketAndKey.substring(bucketEnd + 1)
+ int dataDirectoryEnd = objectKey.indexOf('/data/')
+ assertTrue(dataDirectoryEnd > 0, "Unexpected Iceberg data path:
${committedFile}")
+ String dataObjectPrefix = objectKey.substring(0, dataDirectoryEnd +
'/data/'.length())
+ long objectsBefore = countDataObjects(dataBucket, dataObjectPrefix)
long snapshotsBefore =
(sql """select count(*) from
duplicate_source_target\$snapshots""")[0][0] as long
@@ -126,7 +154,7 @@ suite("test_iceberg_write_merge_duplicate_source_negative",
(sql """select count(*) from
duplicate_source_target\$snapshots""")[0][0] as long)
assertEquals(filesBefore,
(sql """select count(*) from
duplicate_source_target\$files""")[0][0] as long)
- assertEquals(objectsBefore, countDataObjects(dataObjectPath))
+ assertEquals(objectsBefore, countDataObjects(dataBucket, dataObjectPrefix))
order_qt_duplicate_source_atomic_state """
select id, region, payload
from duplicate_source_target
@@ -135,7 +163,7 @@ suite("test_iceberg_write_merge_duplicate_source_negative",
// A sibling delete close can fail only after the data side has closed
successfully. The outer
// MERGE still owns and must remove those unpublished data objects.
- long siblingFailureObjectsBefore = countDataObjects(dataObjectPath)
+ long siblingFailureObjectsBefore = countDataObjects(dataBucket,
dataObjectPrefix)
try {
GetDebugPoint().enableDebugPointForAllBEs("VIcebergDeleteSink.close.inject_failure")
test {
@@ -162,10 +190,11 @@
suite("test_iceberg_write_merge_duplicate_source_negative",
(sql """select count(*) from
duplicate_source_target\$snapshots""")[0][0] as long)
assertEquals(filesBefore,
(sql """select count(*) from
duplicate_source_target\$files""")[0][0] as long)
- assertEquals(siblingFailureObjectsBefore, countDataObjects(dataObjectPath))
+ assertEquals(siblingFailureObjectsBefore, countDataObjects(dataBucket,
dataObjectPrefix))
order_qt_sibling_close_atomic_state """
select id, region, payload
from duplicate_source_target
order by id
"""
+ minioClient.shutdown()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
