This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 623c68480ee branch-3.0: [fix](Export) fix issue that `Export` can not
specify the columns which have capital letters (#43571)
623c68480ee is described below
commit 623c68480eed1c121ce2e91f59ff3de8d341f25a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 12 00:24:46 2024 +0800
branch-3.0: [fix](Export) fix issue that `Export` can not specify the
columns which have capital letters (#43571)
Cherry-picked from #42994
Co-authored-by: Tiewei Fang <[email protected]>
---
.../java/org/apache/doris/analysis/ExportStmt.java | 25 ----------
.../main/java/org/apache/doris/load/ExportJob.java | 4 +-
.../trees/plans/commands/ExportCommand.java | 30 ------------
.../suites/export_p0/test_export_basic.groovy | 55 +++++++++++++++++-----
4 files changed, 45 insertions(+), 69 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index ba7aa50ec69..1ca6c4e3f5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -24,7 +24,6 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
-import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
@@ -50,7 +49,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
-import java.util.stream.Collectors;
// EXPORT statement, export data to dirs by broker.
//
@@ -327,11 +325,6 @@ public class ExportStmt extends StatementBase implements
NotFallbackInParser {
// "" means user specified zero columns
this.columns = properties.getOrDefault(LoadStmt.KEY_IN_PARAM_COLUMNS,
null);
- // check columns are exits
- if (columns != null) {
- checkColumns();
- }
-
// format
this.format =
properties.getOrDefault(LoadStmt.KEY_IN_PARAM_FORMAT_TYPE, "csv").toLowerCase();
@@ -385,24 +378,6 @@ public class ExportStmt extends StatementBase implements
NotFallbackInParser {
this.compressionType = properties.getOrDefault(COMPRESS_TYPE, "");
}
- private void checkColumns() throws DdlException {
- if (this.columns.isEmpty()) {
- throw new DdlException("columns can not be empty");
- }
- Database db =
Env.getCurrentInternalCatalog().getDbOrDdlException(this.tblName.getDb());
- Table table = db.getTableOrDdlException(this.tblName.getTbl());
- List<String> tableColumns = table.getBaseSchema().stream().map(column
-> column.getName())
- .collect(Collectors.toList());
- Splitter split = Splitter.on(',').trimResults().omitEmptyStrings();
-
- List<String> columnsSpecified =
split.splitToList(this.columns.toLowerCase());
- for (String columnName : columnsSpecified) {
- if (!tableColumns.contains(columnName)) {
- throw new DdlException("unknown column [" + columnName + "] in
table [" + this.tblName.getTbl() + "]");
- }
- }
- }
-
@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 584e41d887b..98cfb443ec8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -425,8 +425,8 @@ public class ExportJob implements Writable {
list.addItem(SelectListItem.createStarItem(this.tableName));
} else {
for (Column column : exportTable.getBaseSchema()) {
- String colName = column.getName().toLowerCase();
- if (exportColumns.contains(colName)) {
+ String colName = column.getName();
+ if (exportColumns.contains(colName.toLowerCase())) {
SlotRef slotRef = new SlotRef(this.tableName, colName);
SelectListItem selectListItem = new
SelectListItem(slotRef, null);
list.addItem(selectListItem);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index 38083e406b9..c5d61f58586 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -60,7 +60,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
-import java.util.stream.Collectors;
/**
* EXPORT statement, export data to dirs by broker.
@@ -354,41 +353,12 @@ public class ExportCommand extends Command implements
ForwardWithSync {
private void checkFileProperties(ConnectContext ctx, Map<String, String>
fileProperties, TableName tblName)
throws UserException {
- // check user specified columns
- if (fileProperties.containsKey(LoadStmt.KEY_IN_PARAM_COLUMNS)) {
- checkColumns(ctx,
fileProperties.get(LoadStmt.KEY_IN_PARAM_COLUMNS), tblName);
- }
-
// check user specified label
if (fileProperties.containsKey(LABEL)) {
FeNameFormat.checkLabel(fileProperties.get(LABEL));
}
}
- private void checkColumns(ConnectContext ctx, String columns, TableName
tblName)
- throws AnalysisException, UserException {
- if (columns.isEmpty()) {
- throw new AnalysisException("columns can not be empty");
- }
-
- CatalogIf catalog =
ctx.getEnv().getCatalogMgr().getCatalogOrAnalysisException(tblName.getCtl());
- DatabaseIf db = catalog.getDbOrAnalysisException(tblName.getDb());
- TableIf table = db.getTableOrAnalysisException(tblName.getTbl());
-
- // As for external table
- // their base schemas are equals to full schemas
- List<String> tableColumns = table.getBaseSchema().stream().map(column
-> column.getName())
- .collect(Collectors.toList());
- Splitter split = Splitter.on(',').trimResults().omitEmptyStrings();
-
- List<String> columnsSpecified =
split.splitToList(columns.toLowerCase());
- for (String columnName : columnsSpecified) {
- if (!tableColumns.contains(columnName)) {
- throw new AnalysisException("unknown column [" + columnName +
"] in table [" + tblName.getTbl() + "]");
- }
- }
- }
-
public Map<String, String> getFileProperties() {
return this.fileProperties;
}
diff --git a/regression-test/suites/export_p0/test_export_basic.groovy
b/regression-test/suites/export_p0/test_export_basic.groovy
index 5c736ba264a..ca838232e3f 100644
--- a/regression-test/suites/export_p0/test_export_basic.groovy
+++ b/regression-test/suites/export_p0/test_export_basic.groovy
@@ -66,7 +66,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_export_name} (
`id` int(11) NULL,
- `name` string NULL,
+ `Name` string NULL,
`age` int(11) NULL
)
PARTITION BY RANGE(id)
@@ -135,6 +135,21 @@ suite("test_export_basic", "p0") {
}
}
+ def waiting_export_with_exception = { the_db, export_label, exception_msg
->
+ while (true) {
+ def res = sql """ show export from ${the_db} where label =
"${export_label}" """
+ logger.info("export state: " + res[0][2])
+ if (res[0][2] == "FINISHED") {
+ throw new IllegalStateException("""export finished, do not
contains exception: ${exception_msg}""")
+ } else if (res[0][2] == "CANCELLED") {
+ assertTrue(res[0][10].contains("${exception_msg}"))
+ break;
+ } else {
+ sleep(5000)
+ }
+ }
+ }
+
// 1. basic test
def uuid = UUID.randomUUID().toString()
def outFilePath = """${outfile_path_prefix}_${uuid}"""
@@ -163,7 +178,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_load_name} (
`id` int(11) NULL,
- `name` string NULL,
+ `Name` string NULL,
`age` int(11) NULL
)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
@@ -175,7 +190,7 @@ suite("test_export_basic", "p0") {
table "${table_load_name}"
set 'column_separator', ','
- set 'columns', 'id, name, age'
+ set 'columns', 'id, Name, age'
set 'strict_mode', 'true'
file "${file_path}"
@@ -229,7 +244,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_load_name} (
`id` int(11) NULL,
- `name` string NULL,
+ `Name` string NULL,
`age` int(11) NULL
)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
@@ -241,7 +256,7 @@ suite("test_export_basic", "p0") {
table "${table_load_name}"
set 'column_separator', ','
- set 'columns', 'id, name, age'
+ set 'columns', 'id, Name, age'
set 'strict_mode', 'true'
file "${file_path}"
@@ -295,7 +310,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_load_name} (
`id` int(11) NULL,
- `name` string NULL,
+ `Name` string NULL,
`age` int(11) NULL
)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
@@ -307,7 +322,7 @@ suite("test_export_basic", "p0") {
table "${table_load_name}"
set 'column_separator', ','
- set 'columns', 'id, name, age'
+ set 'columns', 'id, Name, age'
set 'strict_mode', 'true'
file "${file_path}"
@@ -361,7 +376,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_load_name} (
`id` int(11) NULL,
- `name` string NULL,
+ `Name` string NULL,
`age` int(11) NULL
)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
@@ -373,7 +388,7 @@ suite("test_export_basic", "p0") {
table "${table_load_name}"
set 'column_separator', ','
- set 'columns', 'id, name, age'
+ set 'columns', 'id, Name, age'
set 'strict_mode', 'true'
file "${file_path}"
@@ -460,7 +475,7 @@ suite("test_export_basic", "p0") {
"label" = "${label}",
"format" = "csv",
"column_separator"=",",
- "columns" = "id, name",
+ "columns" = "id, Name",
"data_consistency" = "none"
);
"""
@@ -474,7 +489,7 @@ suite("test_export_basic", "p0") {
sql """
CREATE TABLE IF NOT EXISTS ${table_load_name} (
`id` int(11) NULL,
- `name` string NULL
+ `Name` string NULL
)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
"""
@@ -485,7 +500,7 @@ suite("test_export_basic", "p0") {
table "${table_load_name}"
set 'column_separator', ','
- set 'columns', 'id, name'
+ set 'columns', 'id, Name'
set 'strict_mode', 'true'
file "${file_path}"
@@ -641,6 +656,22 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(label_db, label)
+ // test illegal column names
+ uuid = UUID.randomUUID().toString()
+ label = "label_${uuid}"
+
+ sql """
+ EXPORT TABLE ${label_db}.${table_load_name}
+ TO "file://${outFilePath}/"
+ PROPERTIES(
+ "label" = "${label}",
+ "columns" = "col1",
+ "format" = "csv",
+ "column_separator"=",",
+ "data_consistency" = "none"
+ );
+ """
+ waiting_export_with_exception(label_db, label, "Unknown column");
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
delete_files.call("${outFilePath}")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
