This is an automated email from the ASF dual-hosted git repository.
jacktengg pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 914d7b1490a branch-4.2: [fix](test) Fix UUID local outfile and schema
change regressions (#69390)
914d7b1490a is described below
commit 914d7b1490a4c15677938b86e373a82e45b7aa23
Author: TengJianPing <[email protected]>
AuthorDate: Thu Oct 22 09:49:10 2026 +0800
branch-4.2: [fix](test) Fix UUID local outfile and schema change
regressions (#69390)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../datatype_p0/uuid/test_uuid_csv_roundtrip.groovy | 17 ++++++++++++-----
.../suites/datatype_p0/uuid/test_uuid_file_maps.groovy | 17 ++++++++++++-----
.../datatype_p0/uuid/test_uuid_orc_roundtrip.groovy | 17 ++++++++++++-----
.../uuid/test_uuid_orc_varbinary_mapping.groovy | 17 ++++++++++++-----
.../datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy | 17 ++++++++++++-----
.../datatype_p0/uuid/test_uuid_schema_change.groovy | 7 ++++---
6 files changed, 64 insertions(+), 28 deletions(-)
diff --git
a/regression-test/suites/datatype_p0/uuid/test_uuid_csv_roundtrip.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_csv_roundtrip.groovy
index d09733b642d..460debac425 100644
--- a/regression-test/suites/datatype_p0/uuid/test_uuid_csv_roundtrip.groovy
+++ b/regression-test/suites/datatype_p0/uuid/test_uuid_csv_roundtrip.groovy
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-import java.nio.file.Files
import java.nio.file.Paths
// Checklist: G02 G11 G13.
@@ -24,11 +23,16 @@ suite("test_uuid_csv_roundtrip") {
logger.warn("Please set enable_outfile_to_local to true to run
test_uuid_csv_roundtrip")
return
}
- def backend = sql_return_maparray("SHOW BACKENDS").find { it.Alive ==
"true" }
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive ==
"true" }
+ assertFalse(backends.isEmpty())
+ def backend = backends[0]
String securePath = sql_return_maparray(
"SHOW BACKEND CONFIG LIKE 'user_files_secure_path' FROM
${backend.BackendId}")[0].Value
- def outputDirectory = Files.createTempDirectory(Paths.get(securePath),
"test_uuid_csv_roundtrip_")
+ def outputDirectory = Paths.get(securePath,
"test_uuid_csv_roundtrip_${UUID.randomUUID()}")
try {
+ // OUTFILE runs on a BE, which may be remote from the regression
runner.
+ mkdirRemotePathOnAllBE("root", outputDirectory.toString())
+ sql "SET enable_parallel_outfile = false"
sql "DROP TABLE IF EXISTS uuid_file_csv"
sql """CREATE TABLE uuid_file_csv (id INT, u UUID, a ARRAY<UUID>, s
STRUCT<k:UUID>)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
@@ -39,8 +43,11 @@ suite("test_uuid_csv_roundtrip") {
(2, 'ffffffff-ffff-ffff-ffff-ffffffffffff', [], {NULL}),
(3, NULL, NULL, NULL)"""
qt_source "SELECT * FROM uuid_file_csv ORDER BY id"
- sql """SELECT id, u FROM uuid_file_csv ORDER BY id
+ def outfile = sql """SELECT id, u FROM uuid_file_csv ORDER BY id
INTO OUTFILE "file://${outputDirectory}/csv_" FORMAT AS csv"""
+ // Use the BE that actually wrote the file for local TVF reads.
+ backend = backends.find {
outfile[0][3].startsWith("file:///${it.Host}/") }
+ assertNotNull(backend)
String path = "${outputDirectory.fileName}/csv_*"
sql "DROP TABLE IF EXISTS uuid_file_reload_csv"
sql "CREATE TABLE uuid_file_reload_csv LIKE uuid_file_csv"
@@ -49,6 +56,6 @@ suite("test_uuid_csv_roundtrip") {
"backend_id"="${backend.BackendId}")"""
qt_roundtrip "SELECT id, u FROM uuid_file_reload_csv ORDER BY id"
} finally {
- assertTrue(outputDirectory.toFile().deleteDir())
+ deleteRemotePathOnAllBE("root", outputDirectory.toString())
}
}
diff --git a/regression-test/suites/datatype_p0/uuid/test_uuid_file_maps.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_file_maps.groovy
index 3fca80f43b2..c028d2021d6 100644
--- a/regression-test/suites/datatype_p0/uuid/test_uuid_file_maps.groovy
+++ b/regression-test/suites/datatype_p0/uuid/test_uuid_file_maps.groovy
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-import java.nio.file.Files
import java.nio.file.Paths
// Checklist: G08 G09 G10 G13 H02 H08.
@@ -24,11 +23,16 @@ suite("test_uuid_file_maps") {
logger.warn("Please set enable_outfile_to_local to true to run
test_uuid_file_maps")
return
}
- def backend = sql_return_maparray("SHOW BACKENDS").find { it.Alive ==
"true" }
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive ==
"true" }
+ assertFalse(backends.isEmpty())
+ def backend = backends[0]
String securePath = sql_return_maparray(
"SHOW BACKEND CONFIG LIKE 'user_files_secure_path' FROM
${backend.BackendId}")[0].Value
- def outputDirectory = Files.createTempDirectory(Paths.get(securePath),
"test_uuid_file_maps_")
+ def outputDirectory = Paths.get(securePath,
"test_uuid_file_maps_${UUID.randomUUID()}")
try {
+ // OUTFILE runs on a BE, which may be remote from the regression
runner.
+ mkdirRemotePathOnAllBE("root", outputDirectory.toString())
+ sql "SET enable_parallel_outfile = false"
sql "DROP TABLE IF EXISTS uuid_file_maps"
sql """CREATE TABLE uuid_file_maps (id INT,m MAP<UUID,ARRAY<UUID>>,v
MAP<STRING,UUID>)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num'='1')"""
@@ -39,8 +43,11 @@ suite("test_uuid_file_maps") {
(4, {'00000000-0000-0000-0000-000000000000':NULL},
{'zero':'00000000000000000000000000000000'})"""
qt_source "SELECT * FROM uuid_file_maps ORDER BY id"
for (String format : ['parquet', 'orc']) {
- sql """SELECT * FROM uuid_file_maps ORDER BY id
+ def outfile = sql """SELECT * FROM uuid_file_maps ORDER BY id
INTO OUTFILE 'file://${outputDirectory}/${format}_' FORMAT
AS ${format}"""
+ // Use the BE that actually wrote the file for local TVF reads.
+ backend = backends.find {
outfile[0][3].startsWith("file:///${it.Host}/") }
+ assertNotNull(backend)
String path = "${outputDirectory.fileName}/${format}_*"
sql "DROP TABLE IF EXISTS uuid_file_maps_reload"
sql "CREATE TABLE uuid_file_maps_reload LIKE uuid_file_maps"
@@ -52,6 +59,6 @@ suite("test_uuid_file_maps") {
FROM uuid_file_maps_reload ORDER BY id"""
}
} finally {
- assertTrue(outputDirectory.toFile().deleteDir())
+ deleteRemotePathOnAllBE("root", outputDirectory.toString())
}
}
diff --git
a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_roundtrip.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_roundtrip.groovy
index e87d867f2f5..afb5fd2c778 100644
--- a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_roundtrip.groovy
+++ b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_roundtrip.groovy
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-import java.nio.file.Files
import java.nio.file.Paths
// Checklist: G10 G11 G13 H08.
@@ -24,11 +23,16 @@ suite("test_uuid_orc_roundtrip") {
logger.warn("Please set enable_outfile_to_local to true to run
test_uuid_orc_roundtrip")
return
}
- def backend = sql_return_maparray("SHOW BACKENDS").find { it.Alive ==
"true" }
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive ==
"true" }
+ assertFalse(backends.isEmpty())
+ def backend = backends[0]
String securePath = sql_return_maparray(
"SHOW BACKEND CONFIG LIKE 'user_files_secure_path' FROM
${backend.BackendId}")[0].Value
- def outputDirectory = Files.createTempDirectory(Paths.get(securePath),
"test_uuid_orc_roundtrip_")
+ def outputDirectory = Paths.get(securePath,
"test_uuid_orc_roundtrip_${UUID.randomUUID()}")
try {
+ // OUTFILE runs on a BE, which may be remote from the regression
runner.
+ mkdirRemotePathOnAllBE("root", outputDirectory.toString())
+ sql "SET enable_parallel_outfile = false"
sql "DROP TABLE IF EXISTS uuid_file_orc"
sql """CREATE TABLE uuid_file_orc (id INT, u UUID, a ARRAY<UUID>, s
STRUCT<k:UUID>)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
@@ -39,8 +43,11 @@ suite("test_uuid_orc_roundtrip") {
(2, 'ffffffff-ffff-ffff-ffff-ffffffffffff', [], {NULL}),
(3, NULL, NULL, NULL)"""
qt_source "SELECT * FROM uuid_file_orc ORDER BY id"
- sql """SELECT * FROM uuid_file_orc ORDER BY id
+ def outfile = sql """SELECT * FROM uuid_file_orc ORDER BY id
INTO OUTFILE "file://${outputDirectory}/orc_" FORMAT AS orc"""
+ // Use the BE that actually wrote the file for local TVF reads.
+ backend = backends.find {
outfile[0][3].startsWith("file:///${it.Host}/") }
+ assertNotNull(backend)
String path = "${outputDirectory.fileName}/orc_*"
sql "DROP TABLE IF EXISTS uuid_file_reload_orc"
sql "CREATE TABLE uuid_file_reload_orc LIKE uuid_file_orc"
@@ -54,6 +61,6 @@ suite("test_uuid_orc_roundtrip") {
qt_legacy_reader """SELECT * FROM local(
"file_path"="${path}", "format"="orc",
"backend_id"="${backend.BackendId}") ORDER BY id"""
} finally {
- assertTrue(outputDirectory.toFile().deleteDir())
+ deleteRemotePathOnAllBE("root", outputDirectory.toString())
}
}
diff --git
a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy
index 0536041db41..15624ab1e3b 100644
---
a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy
+++
b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-import java.nio.file.Files
import java.nio.file.Paths
suite("test_uuid_orc_varbinary_mapping") {
@@ -23,11 +22,16 @@ suite("test_uuid_orc_varbinary_mapping") {
logger.warn("Please set enable_outfile_to_local to true to run
test_uuid_orc_varbinary_mapping")
return
}
- def backend = sql_return_maparray("SHOW BACKENDS").find { it.Alive ==
"true" }
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive ==
"true" }
+ assertFalse(backends.isEmpty())
+ def backend = backends[0]
String securePath = sql_return_maparray(
"SHOW BACKEND CONFIG LIKE 'user_files_secure_path' FROM
${backend.BackendId}")[0].Value
- def outputDirectory = Files.createTempDirectory(Paths.get(securePath),
"test_uuid_orc_varbinary_")
+ def outputDirectory = Paths.get(securePath,
"test_uuid_orc_varbinary_${UUID.randomUUID()}")
try {
+ // OUTFILE runs on a BE, which may be remote from the regression
runner.
+ mkdirRemotePathOnAllBE("root", outputDirectory.toString())
+ sql "SET enable_parallel_outfile = false"
sql "DROP TABLE IF EXISTS uuid_orc_varbinary_mapping"
sql """CREATE TABLE uuid_orc_varbinary_mapping (
id INT, u UUID, a ARRAY<UUID>, m MAP<UUID,UUID>, s
STRUCT<k:UUID>, b STRING)
@@ -47,9 +51,12 @@ suite("test_uuid_orc_varbinary_mapping") {
{'00112233445566778899aabbccddeeff':'00000000000000000000000000000000'},
{'00000000000000000000000000000000'}, NULL),
(5, NULL, NULL, NULL, NULL, NULL)"""
- sql """SELECT id, u, a, m, s, CAST(b AS VARBINARY) AS b
+ def outfile = sql """SELECT id, u, a, m, s, CAST(b AS VARBINARY) AS b
FROM uuid_orc_varbinary_mapping ORDER BY id
INTO OUTFILE "file://${outputDirectory}/orc_" FORMAT AS ORC"""
+ // Use the BE that actually wrote the file for local TVF reads.
+ backend = backends.find {
outfile[0][3].startsWith("file:///${it.Host}/") }
+ assertNotNull(backend)
// The unconditional native mapping contract is implemented by
FileScanner V2.
for (boolean scannerV2 : [true]) {
sql "SET enable_file_scanner_v2 = ${scannerV2}"
@@ -73,6 +80,6 @@ suite("test_uuid_orc_varbinary_mapping") {
}
}
} finally {
- assertTrue(outputDirectory.toFile().deleteDir())
+ deleteRemotePathOnAllBE("root", outputDirectory.toString())
}
}
diff --git
a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy
index 087abee2a30..2abef8bc57a 100644
--- a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy
+++ b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-import java.nio.file.Files
import java.nio.file.Paths
// Checklist: G08 G09 G11 G13 H08.
@@ -24,11 +23,16 @@ suite("test_uuid_parquet_roundtrip") {
logger.warn("Please set enable_outfile_to_local to true to run
test_uuid_parquet_roundtrip")
return
}
- def backend = sql_return_maparray("SHOW BACKENDS").find { it.Alive ==
"true" }
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive ==
"true" }
+ assertFalse(backends.isEmpty())
+ def backend = backends[0]
String securePath = sql_return_maparray(
"SHOW BACKEND CONFIG LIKE 'user_files_secure_path' FROM
${backend.BackendId}")[0].Value
- def outputDirectory = Files.createTempDirectory(Paths.get(securePath),
"test_uuid_parquet_roundtrip_")
+ def outputDirectory = Paths.get(securePath,
"test_uuid_parquet_roundtrip_${UUID.randomUUID()}")
try {
+ // OUTFILE runs on a BE, which may be remote from the regression
runner.
+ mkdirRemotePathOnAllBE("root", outputDirectory.toString())
+ sql "SET enable_parallel_outfile = false"
sql "DROP TABLE IF EXISTS uuid_file_parquet"
sql """CREATE TABLE uuid_file_parquet (id INT, u UUID, a ARRAY<UUID>,
s STRUCT<k:UUID>)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
@@ -39,8 +43,11 @@ suite("test_uuid_parquet_roundtrip") {
(2, 'ffffffff-ffff-ffff-ffff-ffffffffffff', [], {NULL}),
(3, NULL, NULL, NULL)"""
qt_source "SELECT * FROM uuid_file_parquet ORDER BY id"
- sql """SELECT * FROM uuid_file_parquet ORDER BY id
+ def outfile = sql """SELECT * FROM uuid_file_parquet ORDER BY id
INTO OUTFILE "file://${outputDirectory}/parquet_" FORMAT AS
parquet"""
+ // Use the BE that actually wrote the file for local TVF reads.
+ backend = backends.find {
outfile[0][3].startsWith("file:///${it.Host}/") }
+ assertNotNull(backend)
String path = "${outputDirectory.fileName}/parquet_*"
sql "DROP TABLE IF EXISTS uuid_file_reload_parquet"
sql "CREATE TABLE uuid_file_reload_parquet LIKE uuid_file_parquet"
@@ -52,6 +59,6 @@ suite("test_uuid_parquet_roundtrip") {
order_qt_inferred_types """DESC FUNCTION local(
"file_path"="${path}", "format"="parquet",
"backend_id"="${backend.BackendId}")"""
} finally {
- assertTrue(outputDirectory.toFile().deleteDir())
+ deleteRemotePathOnAllBE("root", outputDirectory.toString())
}
}
diff --git
a/regression-test/suites/datatype_p0/uuid/test_uuid_schema_change.groovy
b/regression-test/suites/datatype_p0/uuid/test_uuid_schema_change.groovy
index f4f5ccbcd1b..db9f450844e 100644
--- a/regression-test/suites/datatype_p0/uuid/test_uuid_schema_change.groovy
+++ b/regression-test/suites/datatype_p0/uuid/test_uuid_schema_change.groovy
@@ -34,7 +34,7 @@ suite("test_uuid_schema_change", "p0") {
"""
waitForSchemaChangeDone {
sql """SHOW ALTER TABLE COLUMN WHERE TableName='uuid_schema_change'
- ORDER BY CreateTime DESC LIMIT 1"""
+ ORDER BY JobId DESC LIMIT 1"""
time 600
}
for (String functionName : ["UUID_V4", "UUID_V7", "GENERATE_UUID_V4",
"GENERATE_UUID_V7"]) {
@@ -62,14 +62,15 @@ suite("test_uuid_schema_change", "p0") {
sql """ALTER TABLE uuid_storage_schema ADD COLUMN added UUID NOT NULL
DEFAULT '00112233-4455-6677-8899-aabbccddeeff'"""
waitForSchemaChangeDone {
- sql "SHOW ALTER TABLE COLUMN WHERE TableName='uuid_storage_schema'
ORDER BY CreateTime DESC LIMIT 1"
+ sql "SHOW ALTER TABLE COLUMN WHERE TableName='uuid_storage_schema'
ORDER BY JobId DESC LIMIT 1"
time 120
}
sql "INSERT INTO uuid_storage_schema(id,u) VALUES (3,
'ffffffff-ffff-ffff-ffff-ffffffffffff')"
order_qt_schema_backfill "SELECT * FROM uuid_storage_schema ORDER BY id"
sql "ALTER TABLE uuid_storage_schema DROP COLUMN added"
+ // ADD and DROP can share a CreateTime; JobId identifies the newest job
unambiguously.
waitForSchemaChangeDone {
- sql "SHOW ALTER TABLE COLUMN WHERE TableName='uuid_storage_schema'
ORDER BY CreateTime DESC LIMIT 1"
+ sql "SHOW ALTER TABLE COLUMN WHERE TableName='uuid_storage_schema'
ORDER BY JobId DESC LIMIT 1"
time 120
}
order_qt_schema_drop "SELECT * FROM uuid_storage_schema ORDER BY id"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
