This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new bf90d918ca6 [fix](regression-test) Make `test_decimal256_outfile_csv`
export the data to S3 rather than local file system (#36592)(#42211) (#42270)
bf90d918ca6 is described below
commit bf90d918ca634529dcbdb562bc7747af5336dfd5
Author: Rayner Chen <[email protected]>
AuthorDate: Tue Oct 22 23:57:30 2024 +0800
[fix](regression-test) Make `test_decimal256_outfile_csv` export the data
to S3 rather than local file system (#36592)(#42211) (#42270)
bp (#36592)(#42211)
---------
Co-authored-by: TengJianPing <[email protected]>
Co-authored-by: Tiewei Fang <[email protected]>
---
.../decimalv3/test_decimal256_outfile_csv.out | 21 ++++++
.../decimalv3/test_decimal256_outfile_csv.groovy | 88 +++++++++-------------
2 files changed, 55 insertions(+), 54 deletions(-)
diff --git
a/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
b/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
index 6dd74d5daa5..e30101a08e6 100644
--- a/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
+++ b/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
@@ -20,3 +20,24 @@
4999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
+-- !select_tvf1 --
+\N \N \N
+-9999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
+-4999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
+-99999999999999999999999999999.999999999 1.0000000000 1.00000000000
+0.000000000 0.0000000000 0.00000000000
+1.000000000
999999999999999999999999999999999999999999999999999999999999999999.9999999999
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+1.000000000
999999999999999999999999999999999999999999999999999999999999999999.9999999999
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+2.000000000
499999999999999999999999999999999999999999999999999999999999999999.9999999999
49999999999999999999999999999999999999999999999999999999999999999.99999999999
+3.000000000
333333333333333333333333333333333333333333333333333333333333333333.3333333333
33333333333333333333333333333333333333333333333333333333333333333.33333333333
+3.000000000
333333333333333333333333333333333333333333333333333333333333333333.3333333333
33333333333333333333333333333333333333333333333333333333333333333.33333333333
+4.000000000
-999999999999999999999999999999999999999999999999999999999999999999.9999999999
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+4.000000000
-999999999999999999999999999999999999999999999999999999999999999999.9999999999
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+5.000000000
-333333333333333333333333333333333333333333333333333333333333333333.3333333333
33333333333333333333333333333333333333333333333333333333333333333.33333333333
+5.000000000
-333333333333333333333333333333333333333333333333333333333333333333.3333333333
33333333333333333333333333333333333333333333333333333333333333333.33333333333
+6.000000000 \N
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+7.000000000 \N
99999999999999999999999999999999999999999999999999999999999999999.99999999999
+99999999999999999999999999999.999999999 1.0000000000 1.00000000000
+4999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
+9999999999999999999999999999999999999999999999999999999999999999999.999999999
1.0000000000 1.00000000000
+
diff --git
a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
index 24054e6408d..736ff970f1c 100644
---
a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
+++
b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
@@ -22,36 +22,15 @@ import java.nio.file.Files
import java.nio.file.Paths
suite("test_decimal256_outfile_csv") {
- StringBuilder strBuilder = new StringBuilder()
- strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser
+ ":" + context.config.jdbcPassword)
- strBuilder.append(" http://"; + context.config.feHttpAddress +
"/rest/v1/config/fe")
-
- String command = strBuilder.toString()
- def process = command.toString().execute()
- def code = process.waitFor()
- def err = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(process.getErrorStream())));
- def out = process.getText()
- logger.info("Request FE Config: code=" + code + ", out=" + out + ", err="
+ err)
- assertEquals(code, 0)
- def response = parseJson(out.trim())
- assertEquals(response.code, 0)
- assertEquals(response.msg, "success")
- def configJson = response.data.rows
- boolean enableOutfileToLocal = false
- for (Object conf: configJson) {
- assert conf instanceof Map
- if (((Map<String, String>) conf).get("Name").toLowerCase() ==
"enable_outfile_to_local") {
- enableOutfileToLocal = ((Map<String, String>)
conf).get("Value").toLowerCase() == "true"
- }
- }
- if (!enableOutfileToLocal) {
- logger.warn("Please set enable_outfile_to_local to true to run
test_outfile")
- return
- }
-
sql "set enable_nereids_planner = true;"
sql "set enable_decimal256 = true;"
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName");
+
sql "DROP TABLE IF EXISTS `test_decimal256_outfile_csv`"
sql """
CREATE TABLE IF NOT EXISTS `test_decimal256_outfile_csv` (
@@ -105,36 +84,37 @@ suite("test_decimal256_outfile_csv") {
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3;
"""
- def uuid = UUID.randomUUID().toString()
- def outFilePath = """/tmp/test_decimal256_outfile_csv_${uuid}"""
- List<List<Object>> backends = sql """ show backends """
- assertTrue(backends.size() > 0)
- if (backends.size() > 1) {
- outFilePath = "/tmp"
+
+ def outFilePath = "${bucket}/outfile/csv/test_decimal256_outfile_csv/exp_"
+
+ def outfile_to_S3 = { export_table_name, foramt ->
+ // select ... into outfile ...
+ def res = sql """
+ SELECT * FROM ${export_table_name} t ORDER BY k1
+ INTO OUTFILE "s3://${outFilePath}"
+ FORMAT AS ${foramt}
+ PROPERTIES (
+ "s3.endpoint" = "${s3_endpoint}",
+ "s3.region" = "${region}",
+ "s3.secret_key"="${sk}",
+ "s3.access_key" = "${ak}"
+ );
+ """
+ return res[0][3]
}
+
try {
logger.info("outfile: " + outFilePath)
- // check outfile
- File path = new File(outFilePath)
- if (!path.exists()) {
- assert path.mkdirs()
- } else {
- throw new IllegalStateException("""${outFilePath} already exists!
""")
- }
- sql """
- SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3 INTO
OUTFILE "file://${outFilePath}/" properties("column_separator" = ",");
- """
- File[] files = path.listFiles()
- assert files.length == 1
- List<String> outLines =
Files.readAllLines(Paths.get(files[0].getAbsolutePath()),
StandardCharsets.UTF_8);
- assert outLines.size() == 19
+ def outfile_url = outfile_to_S3("test_decimal256_outfile_csv", "csv")
+
+ qt_select_tvf1 """ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.csv",
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "region" = "${region}"
+ );
+ """
} finally {
- File path = new File(outFilePath)
- if (path.exists()) {
- for (File f: path.listFiles()) {
- f.delete();
- }
- path.delete();
- }
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
