This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 33053ad1fe [improvement](outfile) support multibyte separator in
outfile clause (#11487)
33053ad1fe is described below
commit 33053ad1fea00c70240e8d32b6d5fad3f2960517
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu Aug 4 09:06:06 2022 +0800
[improvement](outfile) support multibyte separator in outfile clause
(#11487)
---
.../org/apache/doris/analysis/OutFileClause.java | 6 +-
.../data/export/test_outfile_separator.out | 10 ++
.../suites/export/test_outfile_separator.groovy | 102 +++++++++++++++++++++
3 files changed, 116 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 95ef74c5ba..a66bd3b9bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -379,7 +379,7 @@ public class OutFileClause {
if (!isCsvFormat()) {
throw new AnalysisException(PROP_COLUMN_SEPARATOR + " is only
for CSV format");
}
- columnSeparator = properties.get(PROP_COLUMN_SEPARATOR);
+ columnSeparator =
Separator.convertSeparator(properties.get(PROP_COLUMN_SEPARATOR));
processedPropKeys.add(PROP_COLUMN_SEPARATOR);
}
@@ -387,7 +387,7 @@ public class OutFileClause {
if (!isCsvFormat()) {
throw new AnalysisException(PROP_LINE_DELIMITER + " is only
for CSV format");
}
- lineDelimiter = properties.get(PROP_LINE_DELIMITER);
+ lineDelimiter =
Separator.convertSeparator(properties.get(PROP_LINE_DELIMITER));
processedPropKeys.add(PROP_LINE_DELIMITER);
}
@@ -583,3 +583,5 @@ public class OutFileClause {
return sinkOptions;
}
}
+
+
diff --git a/regression-test/data/export/test_outfile_separator.out
b/regression-test/data/export/test_outfile_separator.out
new file mode 100644
index 0000000000..e1ad11da09
--- /dev/null
+++ b/regression-test/data/export/test_outfile_separator.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+1 abc
+2 def
+
+-- !select_2 --
+1 abc
+1 abc
+2 def
+2 def
diff --git a/regression-test/suites/export/test_outfile_separator.groovy
b/regression-test/suites/export/test_outfile_separator.groovy
new file mode 100644
index 0000000000..fb869c5eb0
--- /dev/null
+++ b/regression-test/suites/export/test_outfile_separator.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_outfile", "separator") {
+ StringBuilder strBuilder = new StringBuilder()
+ strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser
+ ":" + context.config.jdbcPassword)
+ strBuilder.append(" http://"; + context.config.feHttpAddress +
"/rest/v1/config/fe")
+
+ String command = strBuilder.toString()
+ def process = command.toString().execute()
+ def code = process.waitFor()
+ def err = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(process.getErrorStream())));
+ def out = process.getText()
+ logger.info("Request FE Config: code=" + code + ", out=" + out + ", err="
+ err)
+ assertEquals(code, 0)
+ def response = parseJson(out.trim())
+ assertEquals(response.code, 0)
+ assertEquals(response.msg, "success")
+ def configJson = response.data.rows
+ boolean enableOutfileToLocal = false
+ for (Object conf: configJson) {
+ assert conf instanceof Map
+ if (((Map<String, String>) conf).get("Name").toLowerCase() ==
"enable_outfile_to_local") {
+ enableOutfileToLocal = ((Map<String, String>)
conf).get("Value").toLowerCase() == "true"
+ }
+ }
+ if (!enableOutfileToLocal) {
+ logger.warn("Please set enable_outfile_to_local to true to run
test_outfile")
+ return
+ }
+ def tableName = "outfile_test_separator"
+ def outFilePath = """${context.file.parent}/tmp_separator"""
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ k1 int, k2 varchar(100)
+ )
+ DISTRIBUTED BY HASH(k1) PROPERTIES("replication_num" = "1");
+ """
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, "abc"), (2, "def");
+ """
+ qt_select_1 """ SELECT * FROM ${tableName} t ORDER BY k1; """
+
+ // check outfile
+ File path = new File(outFilePath)
+ if (!path.exists()) {
+ assert path.mkdirs()
+ } else {
+ throw new IllegalStateException("""${outFilePath} already exists!
""")
+ }
+ sql """
+ SELECT * FROM ${tableName} t INTO OUTFILE "file://${outFilePath}/"
properties("column_separator" = "\\\\x01");
+ """
+ File[] files = path.listFiles()
+ assert files.length == 1
+ List<String> outLines =
Files.readAllLines(Paths.get(files[0].getAbsolutePath()),
StandardCharsets.UTF_8);
+ assert outLines.size() == 2
+
+ streamLoad {
+ db 'regression_test_export'
+ table 'outfile_test_separator'
+ set 'column_separator', '\\x01'
+ file files[0].getAbsolutePath()
+ time 10000 // limit inflight 10s
+ }
+
+ qt_select_2 """ SELECT * FROM ${tableName} t ORDER BY k1; """
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ File path = new File(outFilePath)
+ if (path.exists()) {
+ for (File f: path.listFiles()) {
+ f.delete();
+ }
+ path.delete();
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
