This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fc1b3ecdac4 [feature](regression) add support to run export cases on
multi bes (#55981)
fc1b3ecdac4 is described below
commit fc1b3ecdac455a361e6b876c1acf07d203f5f182
Author: shuke <[email protected]>
AuthorDate: Tue Sep 16 16:24:05 2025 +0800
[feature](regression) add support to run export cases on multi bes (#55981)
---
.../regression/util/RemoteFileOperator.groovy | 370 +++++++++++++++++++++
.../agg_state/test_outfile_agg_state.groovy | 19 +-
.../agg_state_array/test_outfile_agg_array.groovy | 17 +-
.../test_outfile_agg_state_bitmap.groovy | 17 +-
.../nereids_p0/outfile/hll/test_outfile_hll.groovy | 19 +-
.../test_outfile_quantile_state.groovy | 19 +-
6 files changed, 430 insertions(+), 31 deletions(-)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy
new file mode 100644
index 00000000000..13f404f928d
--- /dev/null
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy
@@ -0,0 +1,370 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.util.concurrent.TimeUnit
+import java.security.SecureRandom
+
+/**
+ * A utility class for remote file operations via SSH protocol, supporting
batch operations on multiple hosts.
+ * Core capabilities include remote directory creation, file downloading (via
SCP), and directory deletion.
+ * <p>Key Features:
+ * <ul>
+ * <li>Batch operation support for multiple remote hosts (shared SSH
credentials and port)</li>
+ * <li>Built-in timeout control for all operations to prevent long-term
blocking</li>
+ * <li>Safety check for deletion: only paths containing {@link
#SAFETY_PREFIX} can be deleted (avoids accidental data loss)</li>
+ * <li>Pre-download file count check: skips SCP if remote directory is empty
(prevents "No such file" errors)</li>
+ * </ul>
+ *
+ * <p>Usage Example:
+ * <pre>
+ * // 1. Initialize operator with 2 remote hosts, SSH user "root", port 22,
15s timeout
+ * List<String> hosts = Arrays.asList("172.20.56.7", "172.20.56.14");
+ * RemoteFileOperator operator = new RemoteFileOperator(hosts, "root", 22,
15000);
+ *
+ * // 2. Create directory on all remote hosts
+ * operator.createRemoteDirectories("/tmp/doristest_data");
+ *
+ * // 3. Download files from remote to local (skips if remote is empty)
+ * operator.scpToLocal("/tmp/doristest_data", "/local/data");
+ *
+ * // 4. Delete remote directory (only allowed if path contains "doristest")
+ * operator.deleteRemoteDirectories("/tmp/doristest_data");
+ * </pre>
+ */
+class RemoteFileOperator {
+ private static final Logger logger =
LoggerFactory.getLogger(RemoteFileOperator.class)
+ public static final String SAFETY_PREFIX = "doristest"
+
+ private List<String> hosts
+ private String username
+ private int port
+ private int timeout
+
+ /**
+ * Constructor for RemoteFileOperator
+ *
+ * @param hosts Single host string or list of host strings
+ * @param username SSH username (shared across all hosts)
+ * @param port SSH port (shared across all hosts), default is 22
+ * @param timeout Operation timeout in milliseconds, default is 10000
+ */
+ RemoteFileOperator(def hosts, String username, int port = 22, int timeout
= 10000) {
+ if (hosts instanceof List) {
+ this.hosts = hosts
+ } else if (hosts instanceof String) {
+ this.hosts = [hosts]
+ } else {
+ throw new IllegalArgumentException("Hosts must be a string or list
of strings")
+ }
+
+ this.username = username
+ this.port = port
+ this.timeout = timeout
+ }
+
+ /**
+ * Create directories on all remote hosts
+ * Throws exception if any host fails
+ *
+ * @param dirPath Path of the directory to create
+ * @throws Exception if directory creation fails on any host
+ */
+ void createRemoteDirectories(String dirPath) throws Exception {
+ hosts.each { host ->
+ logger.info("Processing host: ${host}")
+ def command = "mkdir -p ${escapePath(dirPath)}"
+ def execResult = executeSshCommand(host, command)
+
+ if (execResult.timedOut) {
+ def errorMsg = "Timeout creating directory on ${host}
(${timeout}ms)"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ if (execResult.exitCode != 0) {
+ def errorMsg = "Failed to create directory on ${host} (exit
code: ${execResult.exitCode}): ${execResult.error}"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+ }
+
+ logger.info("Successfully created directory '${dirPath}' on all hosts")
+ }
+
+ /**
+ * Download files from all remote hosts directly to local base directory
+ * without host-specific subfolders
+ * Throws exception if any host fails
+ *
+ * @param remoteDir Remote directory path to download
+ * @param localBaseDir Local directory where all files will be copied
+ * @throws Exception if SCP fails on any host
+ */
+ void scpToLocal(String remoteDir, String localBaseDir) throws Exception {
+ logger.info("Starting SCP download process from ${hosts.size()} hosts
to ${localBaseDir}")
+ remoteDir = remoteDir.trim()
+ localBaseDir = localBaseDir.trim()
+ if (remoteDir == null || remoteDir.trim().isEmpty()) {
+ throw new IllegalArgumentException("Remote directory cannot be
null or empty")
+ }
+ if (localBaseDir == null || localBaseDir.trim().isEmpty()) {
+ throw new IllegalArgumentException("Local base directory cannot be
null or empty")
+ }
+
+ // Create base directory if it doesn't exist
+ def baseDir = new File(localBaseDir)
+ if (!baseDir.exists() && !baseDir.mkdirs()) {
+ def errorMsg = "Failed to create local base directory:
${localBaseDir}"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ hosts.each { host ->
+ // Step 1: Check if remote directory has files (count regular
files only)
+ // scp user@host:/tmp/doristest/* will fail if doristest has no
files.
+ String countCommand = """ssh -p ${port} ${username}@${host} "find
${escapePath(remoteDir)} -maxdepth 1 -type f | wc -l" """
+ def countResult = executeLocalCommand(countCommand)
+
+ if (countResult.timedOut) {
+ def errorMsg = "Timeout checking file count on ${host}
(${timeout}ms)"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ if (countResult.exitCode != 0) {
+ def errorMsg = "Failed to check file count on ${host} (exit
code: ${countResult.exitCode}): ${countResult.error}"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ // Parse file count (handle possible whitespace in output)
+ int fileCount = countResult.stdout.trim().toInteger() // wc -l
output is in error stream due to redirect
+ logger.info("Found ${fileCount} files in ${host}:${remoteDir}")
+
+ // Step 2: Only execute SCP if there are files to copy
+ if (fileCount > 0) {
+ logger.info("Downloading ${fileCount} files from
${host}:${remoteDir} to ${localBaseDir}")
+
+ def normalizedRemoteDir = (remoteDir.endsWith('/') ? remoteDir
: "${remoteDir}/") + "*"
+ def scpCommand = "scp -P ${port}
${username}@${host}:${escapePath(normalizedRemoteDir)}
${escapePath(localBaseDir)}"
+ def execResult = executeLocalCommand(scpCommand)
+
+ if (execResult.timedOut) {
+ def errorMsg = "Timeout downloading from ${host}
(${timeout}ms)"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ if (execResult.exitCode != 0) {
+ def errorMsg = "Failed to download from ${host} (exit
code: ${execResult.exitCode}): ${execResult.error}"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+ logger.info("Successfully downloaded ${fileCount} files from
${host}")
+ } else {
+ logger.info("No files found in ${host}:${remoteDir}, skipping
SCP")
+ }
+ }
+
+ logger.info("SCP download process completed for all hosts")
+ }
+
+ /**
+ * Delete directories on all remote hosts
+ * Throws exception if any host fails
+ *
+ * @param dirPath Path of the directory to delete
+ * @throws Exception if directory deletion fails on any host
+ */
+ void deleteRemoteDirectories(String dirPath) throws Exception {
+ logger.info("Deleting directory '${dirPath}' on ${hosts.size()} hosts")
+ if (!dirPath.contains(SAFETY_PREFIX)) {
+ def errorMsg = "Deletion forbidden: Path '${dirPath}' does not
contain safety prefix '${SAFETY_PREFIX}'"
+ logger.error(errorMsg)
+ throw new SecurityException(errorMsg);
+ }
+
+ hosts.each { host ->
+ logger.info("Processing host: ${host}")
+ def command = "rm -rf ${escapePath(dirPath)}"
+ def execResult = executeSshCommand(host, command)
+
+ if (execResult.timedOut) {
+ def errorMsg = "Timeout deleting directory on ${host}
(${timeout}ms)"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+
+ if (execResult.exitCode != 0) {
+ def errorMsg = "Failed to delete directory on ${host} (exit
code: ${execResult.exitCode}): ${execResult.error}"
+ logger.error(errorMsg)
+ throw new Exception(errorMsg)
+ }
+ }
+
+ logger.info("Successfully deleted directory '${dirPath}' on all hosts")
+ }
+
+ private Map executeSshCommand(String host, String command) {
+ def sshCommand = "ssh -p ${port} ${username}@${host} '${command}'"
+ return executeLocalCommand(sshCommand)
+ }
+
+ private Map executeLocalCommand(String command) {
+ def result = [
+ exitCode: -1,
+ stdout: '',
+ stderr: '',
+ timedOut: false
+ ]
+
+ try {
+ logger.debug("Executing command: ${command}")
+ Process process = new ProcessBuilder('/bin/sh', '-c', command)
+ .redirectErrorStream(false)
+ .start()
+
+ boolean completed = process.waitFor(timeout, TimeUnit.MILLISECONDS)
+
+ if (!completed) {
+ process.destroyForcibly()
+ result.timedOut = true
+ logger.warn("Command timed out after ${timeout}ms: ${command}")
+ return result
+ }
+
+ result.exitCode = process.exitValue()
+ result.stdout = process.inputStream.text.trim()
+ result.stderr = process.errorStream.text.trim()
+
+ } catch (Exception e) {
+ result.stderr = e.message
+ logger.error("Error executing command: ${e.message}", e)
+ }
+
+ return result
+ }
+
+ private String escapePath(String path) {
+ return "'${path.replace("'", "'\"'\"'")}'"
+ }
+}
+
+class UniquePathGenerator {
+ private static final SecureRandom random = new SecureRandom()
+ private static final String RANDOM_CHARS =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ private static final int RANDOM_LENGTH = 6 // Adjust length for more/less
uniqueness
+
+ /**
+ * Generates a short, unique local path with "doristest_" prefix
+ * Format: doristest_<timestamp>_<random_chars>
+ *
+ * @param baseDir (Optional) Base directory to prepend, null for current
working directory
+ * @return Unique path string
+ */
+ static String generateUniqueLocalPath(String baseDir, String prefix) {
+ // Generate random characters
+ String randomStr = (1..RANDOM_LENGTH).collect {
+ RANDOM_CHARS[random.nextInt(RANDOM_CHARS.length())]
+ }.join()
+
+ // Create base filename
+ String fileName = "${prefix}_${randomStr}"
+
+ // Combine with base directory if provided
+ if (baseDir) {
+ return new File(baseDir, fileName).absolutePath
+ }
+
+ return new File(fileName).absolutePath
+ }
+}
+
+/**
+ * Test helper class for managing temporary files/directories during export
regression tests.
+ * Automates the creation, collection, and cleanup of temporary directories
(both local and remote).
+ * <p>Key Responsibilities:
+ * <ul>
+ * <li>Generate unique local/remote temporary directories using {@link
UniquePathGenerator}</li>
+ * <li>Handle remote directory creation via {@link RemoteFileOperator}</li>
+ * <li>Facilitate file collection from remote hosts to local directory</li>
+ * <li>Automatically clean up temporary resources (local/remote) via {@link
AutoCloseable} interface</li>
+ * </ul>
+ *
+ * <p>Lifecycle:
+ * <ol>
+ * <li>On initialization: Creates unique local and remote directories,
initializes SSH operator</li>
+ * <li>During test: Use {@link #collect()} to download files from remote to
local directory</li>
+ * <li>On cleanup (via try-with-resources or explicit close()): Deletes
remote directory and local directory</li>
+ * </ol>
+ *
+ * <p>Usage Example:
+ * <pre>
+ * // Initialize with target hosts
+ * List<String> testHosts = Arrays.asList("192.168.1.10", "192.168.1.11");
+ *
+ * // Use try-with-resources to ensure automatic cleanup
+ * try (ExportTestHelper testHelper = new ExportTestHelper(testHosts)) {
+ * // Test logic that generates files in testHelper.remoteDir on remote
hosts
+ * runExportTest(testHelper.remoteDir);
+ *
+ * // Collect generated files to local directory
+ * testHelper.collect();
+ *
+ * // Verify exported files in testHelper.localDir
+ * assertExportedFiles(testHelper.localDir);
+ * }
+ * // Cleanup happens automatically here: remote and local dirs are deleted
+ * </pre>
+ */
+class ExportTestHelper implements AutoCloseable {
+ private static final Logger logger =
LoggerFactory.getLogger(ExportTestHelper.class)
+ String localDir
+ String remoteDir
+ RemoteFileOperator operator
+ boolean deleteTmpFile = false
+
+ public ExportTestHelper(List<String> hosts) {
+ localDir = UniquePathGenerator.generateUniqueLocalPath("/tmp",
RemoteFileOperator.SAFETY_PREFIX+"_l")
+ remoteDir = UniquePathGenerator.generateUniqueLocalPath("/tmp",
RemoteFileOperator.SAFETY_PREFIX)
+
+ operator = new RemoteFileOperator(hosts, "root")
+ operator.createRemoteDirectories(remoteDir)
+ deleteTmpFile = true
+ }
+
+ public void collect() {
+ operator.scpToLocal(remoteDir, localDir)
+ }
+
+ @Override
+ void close() throws Exception {
+ if (deleteTmpFile && operator) {
+ operator.deleteRemoteDirectories(remoteDir)
+ def localDirFile = new File(localDir);
+ if (localDirFile.exists() &&
localDir.contains(RemoteFileOperator.SAFETY_PREFIX)) {
+ if (localDirFile.deleteDir()) {
+ logger.info("Successfully deleted local temporary
directory: ${localDir}");
+ }
+ }
+ }
+ }
+}
diff --git
a/regression-test/suites/nereids_p0/outfile/agg_state/test_outfile_agg_state.groovy
b/regression-test/suites/nereids_p0/outfile/agg_state/test_outfile_agg_state.groovy
index e9c51876843..6ac2f90914a 100644
---
a/regression-test/suites/nereids_p0/outfile/agg_state/test_outfile_agg_state.groovy
+++
b/regression-test/suites/nereids_p0/outfile/agg_state/test_outfile_agg_state.groovy
@@ -16,16 +16,18 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
-
+import org.apache.doris.regression.util.ExportTestHelper
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_agg_state") {
- def outFilePath = """./tmp/test_outfile_agg_state"""
- File path = new File(outFilePath)
- path.deleteDir()
- path.mkdirs()
+ def hosts = []
+ List<List<Object>> backends = sql("show backends");
+ for (def b : backends) {
+ hosts.add(b[1])
+ }
+ ExportTestHelper testHelper = new ExportTestHelper(hosts)
sql "set enable_agg_state=true"
sql "DROP TABLE IF EXISTS a_table"
@@ -45,7 +47,8 @@ suite("test_outfile_agg_state") {
qt_test "select k1,max_by_merge(k2),group_concat_merge(k3) from a_table
group by k1 order by k1;"
- sql """select * from a_table into outfile
"file://${path.getAbsolutePath()}/tmp" FORMAT AS PARQUET;"""
+ sql """select * from a_table into outfile
"file://${testHelper.remoteDir}/e_" FORMAT AS PARQUET;"""
+ testHelper.collect()
sql "DROP TABLE IF EXISTS a_table2"
sql """
@@ -59,10 +62,12 @@ suite("test_outfile_agg_state") {
properties("replication_num" = "1");
"""
- def filePath=path.getAbsolutePath()+"/tmp*"
+ def filePath=testHelper.localDir+"/*"
cmd """
curl --location-trusted -u
${context.config.jdbcUser}:${context.config.jdbcPassword} -H "format:PARQUET"
-H "Expect:100-continue" -T ${filePath} -XPUT
http://${context.config.feHttpAddress}/api/regression_test_nereids_p0_outfile_agg_state/a_table2/_stream_load
"""
Thread.sleep(10000)
qt_test "select k1,max_by_merge(k2),group_concat_merge(k3) from a_table2
group by k1 order by k1;"
+
+ testHelper.close()
}
diff --git
a/regression-test/suites/nereids_p0/outfile/agg_state_array/test_outfile_agg_array.groovy
b/regression-test/suites/nereids_p0/outfile/agg_state_array/test_outfile_agg_array.groovy
index a7d0992f47c..e7908dd4df6 100644
---
a/regression-test/suites/nereids_p0/outfile/agg_state_array/test_outfile_agg_array.groovy
+++
b/regression-test/suites/nereids_p0/outfile/agg_state_array/test_outfile_agg_array.groovy
@@ -16,16 +16,19 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.ExportTestHelper
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_agg_state_array") {
- def outFilePath = """./tmp/test_outfile_agg_state_array"""
- File path = new File(outFilePath)
- path.deleteDir()
- path.mkdirs()
+ def hosts = []
+ List<List<Object>> backends = sql("show backends");
+ for (def b : backends) {
+ hosts.add(b[1])
+ }
+ ExportTestHelper testHelper = new ExportTestHelper(hosts)
sql "set enable_agg_state=true"
sql "DROP TABLE IF EXISTS a_table"
@@ -44,7 +47,8 @@ suite("test_outfile_agg_state_array") {
qt_test "select k1,array_agg_merge(k2) from a_table group by k1 order by
k1;"
- sql """select * from a_table into outfile
"file://${path.getAbsolutePath()}/tmp" FORMAT AS PARQUET;"""
+ sql """select * from a_table into outfile
"file://${testHelper.remoteDir}/tmp_" FORMAT AS PARQUET;"""
+ testHelper.collect()
sql "DROP TABLE IF EXISTS a_table2"
sql """
@@ -58,10 +62,11 @@ suite("test_outfile_agg_state_array") {
properties("replication_num" = "1");
"""
- def filePath=path.getAbsolutePath()+"/tmp*"
+ def filePath=testHelper.localDir+"/tmp_*"
cmd """
curl --location-trusted -u
${context.config.jdbcUser}:${context.config.jdbcPassword} -H "format:PARQUET"
-H "Expect:100-continue" -T ${filePath}
http://${context.config.feHttpAddress}/api/regression_test_nereids_p0_outfile_agg_state_array/a_table2/_stream_load
"""
Thread.sleep(10000)
qt_test "select k1,max_by_merge(k2),group_concat_merge(k3) from a_table2
group by k1 order by k1;"
+ testHelper.close()
}
diff --git
a/regression-test/suites/nereids_p0/outfile/agg_state_bitmap/test_outfile_agg_state_bitmap.groovy
b/regression-test/suites/nereids_p0/outfile/agg_state_bitmap/test_outfile_agg_state_bitmap.groovy
index d46dd7f15ee..fc845f53521 100644
---
a/regression-test/suites/nereids_p0/outfile/agg_state_bitmap/test_outfile_agg_state_bitmap.groovy
+++
b/regression-test/suites/nereids_p0/outfile/agg_state_bitmap/test_outfile_agg_state_bitmap.groovy
@@ -16,16 +16,19 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.ExportTestHelper
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_agg_state_bitmap") {
- def outFilePath = """./tmp/test_outfile_agg_state_bitmap"""
- File path = new File(outFilePath)
- path.deleteDir()
- path.mkdirs()
+ def hosts = []
+ List<List<Object>> backends = sql("show backends");
+ for (def b : backends) {
+ hosts.add(b[1])
+ }
+ ExportTestHelper testHelper = new ExportTestHelper(hosts)
sql "set enable_agg_state=true"
sql "DROP TABLE IF EXISTS a_table"
@@ -44,7 +47,8 @@ suite("test_outfile_agg_state_bitmap") {
qt_test "select k1,bitmap_to_string(bitmap_union_merge(k2)) from a_table
group by k1 order by k1;"
- sql """select * from a_table into outfile
"file://${path.getAbsolutePath()}/tmp" FORMAT AS PARQUET;"""
+ sql """select * from a_table into outfile
"file://${testHelper.remoteDir}/tmp_" FORMAT AS PARQUET;"""
+ testHelper.collect()
sql "DROP TABLE IF EXISTS a_table2"
sql """
@@ -57,10 +61,11 @@ suite("test_outfile_agg_state_bitmap") {
properties("replication_num" = "1");
"""
- def filePath=path.getAbsolutePath()+"/tmp*"
+ def filePath=testHelper.localDir+"/tmp_*"
cmd """
curl --location-trusted -u
${context.config.jdbcUser}:${context.config.jdbcPassword} -H "format:PARQUET"
-H "Expect:100-continue" -T ${filePath}
http://${context.config.feHttpAddress}/api/regression_test_nereids_p0_outfile_agg_state_bitmap/a_table2/_stream_load
"""
Thread.sleep(10000)
qt_test "select k1,bitmap_to_string(bitmap_union_merge(k2)) from a_table
group by k1 order by k1;"
+ testHelper.close()
}
diff --git
a/regression-test/suites/nereids_p0/outfile/hll/test_outfile_hll.groovy
b/regression-test/suites/nereids_p0/outfile/hll/test_outfile_hll.groovy
index 4d8ff939dba..ab7d7616f1b 100644
--- a/regression-test/suites/nereids_p0/outfile/hll/test_outfile_hll.groovy
+++ b/regression-test/suites/nereids_p0/outfile/hll/test_outfile_hll.groovy
@@ -16,6 +16,7 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.ExportTestHelper
import java.nio.charset.StandardCharsets
import java.nio.file.Files
@@ -23,10 +24,13 @@ import java.nio.file.Paths
suite("test_outfile_hll") {
sql "set return_object_data_as_binary=true"
- def outFilePath = """./tmp/test_outfile_hll"""
- File path = new File(outFilePath)
- path.deleteDir()
- path.mkdirs()
+
+ def hosts = []
+ List<List<Object>> backends = sql("show backends");
+ for (def b : backends) {
+ hosts.add(b[1])
+ }
+ ExportTestHelper testHelper = new ExportTestHelper(hosts)
sql "DROP TABLE IF EXISTS h_table"
sql """
@@ -45,7 +49,8 @@ suite("test_outfile_hll") {
qt_test "select k1,hll_union_agg(k2) from h_table group by k1 order by k1;"
- sql """select k1, cast(hll_to_base64(k2) as string) as tmp from h_table
into outfile "file://${path.getAbsolutePath()}/tmp" FORMAT AS PARQUET;"""
+ sql """select k1, cast(hll_to_base64(k2) as string) as tmp from h_table
into outfile "file://${testHelper.remoteDir}/tmp_" FORMAT AS PARQUET;"""
+ testHelper.collect()
sql "DROP TABLE IF EXISTS h_table2"
sql """
@@ -58,10 +63,12 @@ suite("test_outfile_hll") {
properties("replication_num" = "1");
"""
- def filePath=path.getAbsolutePath()+"/tmp*"
+ def filePath=testHelper.localDir+"/tmp_*"
cmd """
curl --location-trusted -u
${context.config.jdbcUser}:${context.config.jdbcPassword} -H "columns: k1, tmp,
k2=hll_from_base64(tmp)" -H "format:PARQUET" -H "Expect:100-continue" -T
${filePath} -XPUT
http://${context.config.feHttpAddress}/api/regression_test_nereids_p0_outfile_hll/h_table2/_stream_load
"""
Thread.sleep(10000)
qt_test "select k1,hll_union_agg(k2) from h_table2 group by k1 order by
k1;"
+
+ testHelper.close()
}
diff --git
a/regression-test/suites/nereids_p0/outfile/quantile_state/test_outfile_quantile_state.groovy
b/regression-test/suites/nereids_p0/outfile/quantile_state/test_outfile_quantile_state.groovy
index 5b9aee31c5e..3108a4f7715 100644
---
a/regression-test/suites/nereids_p0/outfile/quantile_state/test_outfile_quantile_state.groovy
+++
b/regression-test/suites/nereids_p0/outfile/quantile_state/test_outfile_quantile_state.groovy
@@ -16,18 +16,22 @@
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.ExportTestHelper
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_quantile_state") {
+ def hosts = []
+ List<List<Object>> backends = sql("show backends");
+ for (def b : backends) {
+ hosts.add(b[1])
+ }
+ ExportTestHelper testHelper = new ExportTestHelper(hosts)
+
sql "set enable_agg_state=true"
sql "set return_object_data_as_binary=true"
- def outFilePath = """./tmp/test_outfile_quantile_state"""
- File path = new File(outFilePath)
- path.deleteDir()
- path.mkdirs()
sql "DROP TABLE IF EXISTS q_table"
sql """
@@ -46,7 +50,8 @@ suite("test_outfile_quantile_state") {
qt_test "select k1,quantile_percent(quantile_union(k2),0.5) from q_table
group by k1 order by k1;"
- sql """select k1, quantile_union_state(k2) as k2 from q_table into outfile
"file://${path.getAbsolutePath()}/tmp" FORMAT AS PARQUET;"""
+ sql """select k1, quantile_union_state(k2) as k2 from q_table into outfile
"file://${testHelper.remoteDir}/tmp_" FORMAT AS PARQUET;"""
+ testHelper.collect()
sql "DROP TABLE IF EXISTS q_table2"
sql """
@@ -59,10 +64,12 @@ suite("test_outfile_quantile_state") {
properties("replication_num" = "1");
"""
- def filePath=path.getAbsolutePath()+"/tmp*"
+ def filePath=testHelper.localDir+"/tmp_*"
cmd """
curl --location-trusted -u
${context.config.jdbcUser}:${context.config.jdbcPassword} -H "format:PARQUET"
-H "Expect:100-continue" -T ${filePath}
http://${context.config.feHttpAddress}/api/regression_test_nereids_p0_outfile_quantile_state/q_table2/_stream_load
"""
Thread.sleep(10000)
qt_test "select k1,quantile_percent(quantile_union_merge(k2),0.5) from
q_table2 group by k1 order by k1;"
+
+ testHelper.close()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
