This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f978079830a [fix](Export) Set the default value of the
`data_consistence` property of export to `partition` (#32830)
f978079830a is described below
commit f978079830aa5e5436120e03b283f9bbd0b0784a
Author: Tiewei Fang <[email protected]>
AuthorDate: Wed Apr 3 17:12:40 2024 +0800
[fix](Export) Set the default value of the `data_consistence` property of
export to `partition` (#32830)
---
.../java/org/apache/doris/analysis/ExportStmt.java | 18 ++++++------
.../main/java/org/apache/doris/load/ExportJob.java | 2 +-
.../trees/plans/commands/ExportCommand.java | 16 ++++++-----
.../analysis/ExportToOutfileLogicalPlanTest.java | 20 +++++++++----
.../suites/export_p0/test_export_basic.groovy | 33 ++++++++++++++--------
.../export_p0/test_export_data_consistency.groovy | 3 +-
6 files changed, 57 insertions(+), 35 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index b7a3a84a826..681f6345486 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -106,7 +106,7 @@ public class ExportStmt extends StatementBase {
private String maxFileSize;
private String deleteExistingFiles;
private String withBom;
- private String dataConsistency;
+ private String dataConsistency = ExportJob.CONSISTENT_PARTITION;
private SessionVariable sessionVariables;
private String qualifiedUser;
@@ -365,14 +365,16 @@ public class ExportStmt extends StatementBase {
this.withBom = properties.getOrDefault(OutFileClause.PROP_WITH_BOM,
"false");
// data consistency
- String dataConsistencyStr = properties.get(DATA_CONSISTENCY);
- if (dataConsistencyStr != null) {
- if
(!dataConsistencyStr.equalsIgnoreCase(ExportJob.CONSISTENT_PARTITION)) {
- throw new UserException("The value of data_consistency is
invalid, only `partition` is allowed");
+ if (properties.containsKey(DATA_CONSISTENCY)) {
+ String dataConsistencyStr = properties.get(DATA_CONSISTENCY);
+ if
(ExportJob.CONSISTENT_NONE.equalsIgnoreCase(dataConsistencyStr)) {
+ this.dataConsistency = ExportJob.CONSISTENT_NONE;
+ } else if
(ExportJob.CONSISTENT_PARTITION.equalsIgnoreCase(dataConsistencyStr)) {
+ this.dataConsistency = ExportJob.CONSISTENT_PARTITION;
+ } else {
+ throw new AnalysisException("The value of data_consistency is
invalid, please use `"
+ + ExportJob.CONSISTENT_PARTITION + "`/`" +
ExportJob.CONSISTENT_NONE + "`");
}
- this.dataConsistency = ExportJob.CONSISTENT_PARTITION;
- } else {
- this.dataConsistency = ExportJob.CONSISTENT_NONE;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index cc47c641b0c..d0ccf23ae0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -228,7 +228,7 @@ public class ExportJob implements Writable {
this.lineDelimiter = "\n";
this.columns = "";
this.withBom = "false";
- this.dataConsistency = "all";
+ this.dataConsistency = CONSISTENT_PARTITION;
}
public ExportJob(long jobId) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index b872f399fad..aed2b3e24db 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -313,14 +313,16 @@ public class ExportCommand extends Command implements
ForwardWithSync {
exportJob.setUserIdentity(ctx.getCurrentUserIdentity());
// set data consistency
- String dataConsistencyStr = fileProperties.get(DATA_CONSISTENCY);
- if (dataConsistencyStr != null) {
- if
(!dataConsistencyStr.equalsIgnoreCase(ExportJob.CONSISTENT_PARTITION)) {
- throw new AnalysisException("The value of data_consistency is
invalid, only partition is allowed!");
+ if (fileProperties.containsKey(DATA_CONSISTENCY)) {
+ String dataConsistencyStr = fileProperties.get(DATA_CONSISTENCY);
+ if
(ExportJob.CONSISTENT_NONE.equalsIgnoreCase(dataConsistencyStr)) {
+ exportJob.setDataConsistency(ExportJob.CONSISTENT_NONE);
+ } else if
(ExportJob.CONSISTENT_PARTITION.equalsIgnoreCase(dataConsistencyStr)) {
+ exportJob.setDataConsistency(ExportJob.CONSISTENT_PARTITION);
+ } else {
+ throw new AnalysisException("The value of data_consistency is
invalid, please use `"
+ + ExportJob.CONSISTENT_PARTITION + "`/`" +
ExportJob.CONSISTENT_NONE + "`");
}
- exportJob.setDataConsistency(ExportJob.CONSISTENT_PARTITION);
- } else {
- exportJob.setDataConsistency(ExportJob.CONSISTENT_NONE);
}
// Must copy session variable, because session variable may be changed
during export job running.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java
index 633ab0cdd47..43256a14a13 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExportToOutfileLogicalPlanTest.java
@@ -79,7 +79,10 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
public void testNormal() throws UserException {
// The origin export sql
String exportSql = "EXPORT TABLE testDb.table1\n"
- + "TO \"file:///tmp/exp_\";";
+ + "TO \"file:///tmp/exp_\" "
+ + "PROPERTIES(\n"
+ + "\"data_consistency\" = \"none\"\n"
+ + ");";
List<Long> currentTablets1 = Arrays.asList(10010L, 10012L, 10014L,
10016L, 10018L, 10020L, 10022L, 10024L,
10026L, 10028L);
@@ -126,7 +129,8 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
String exportSql = "EXPORT TABLE testDb.table1\n"
+ "TO \"file:///tmp/exp_\" "
+ "PROPERTIES(\n"
- + "\"parallelism\" = \"4\"\n"
+ + "\"parallelism\" = \"4\",\n"
+ + "\"data_consistency\" = \"none\"\n"
+ ");";
// This export sql should generate 4 array, and there should be 1
outfile sql in per array.
@@ -180,7 +184,8 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
String exportSql = "EXPORT TABLE testDb.table1\n"
+ "TO \"file:///tmp/exp_\" "
+ "PROPERTIES(\n"
- + "\"parallelism\" = \"3\"\n"
+ + "\"parallelism\" = \"3\",\n"
+ + "\"data_consistency\" = \"none\"\n"
+ ");";
// This export sql should generate 4 array, and there should be 1
outfile sql in per array.
@@ -240,7 +245,8 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
String exportSql = "EXPORT TABLE testDb.table1 PARTITION (p1)\n"
+ "TO \"file:///tmp/exp_\" "
+ "PROPERTIES(\n"
- + "\"parallelism\" = \"4\"\n"
+ + "\"parallelism\" = \"4\",\n"
+ + "\"data_consistency\" = \"none\"\n"
+ ");";
// This export sql should generate 4 array, and there should be 1
outfile sql in per array.
@@ -293,7 +299,8 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
String exportSql = "EXPORT TABLE testDb.table1 PARTITION (p1, p4)\n"
+ "TO \"file:///tmp/exp_\" "
+ "PROPERTIES(\n"
- + "\"parallelism\" = \"4\"\n"
+ + "\"parallelism\" = \"4\",\n"
+ + "\"data_consistency\" = \"none\"\n"
+ ");";
// This export sql should generate 4 array, and there should be 1
outfile sql in per array.
@@ -344,7 +351,8 @@ public class ExportToOutfileLogicalPlanTest extends
TestWithFeService {
String exportSql = "EXPORT TABLE testDb.table1 PARTITION (p1)\n"
+ "TO \"file:///tmp/exp_\" "
+ "PROPERTIES(\n"
- + "\"parallelism\" = \"20\"\n"
+ + "\"parallelism\" = \"20\",\n"
+ + "\"data_consistency\" = \"none\"\n"
+ ");";
// This export sql should generate 10 array because parallelism is
less than the number of tablets,
diff --git a/regression-test/suites/export_p0/test_export_basic.groovy
b/regression-test/suites/export_p0/test_export_basic.groovy
index d4cd9329c92..5c736ba264a 100644
--- a/regression-test/suites/export_p0/test_export_basic.groovy
+++ b/regression-test/suites/export_p0/test_export_basic.groovy
@@ -149,7 +149,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -214,7 +215,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -279,7 +281,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -344,7 +347,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -411,7 +415,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label1}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
sql """
@@ -420,7 +425,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label2}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label1)
@@ -454,7 +460,8 @@ suite("test_export_basic", "p0") {
"label" = "${label}",
"format" = "csv",
"column_separator"=",",
- "columns" = "id, name"
+ "columns" = "id, name",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -519,7 +526,8 @@ suite("test_export_basic", "p0") {
"label" = "${label}",
"format" = "csv",
"column_separator"=",",
- "columns" = "id"
+ "columns" = "id",
+ "data_consistency" = "none"
);
"""
waiting_export.call(db, label)
@@ -588,7 +596,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(label_db, label)
@@ -601,7 +610,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
exception "has already been used"
@@ -625,7 +635,8 @@ suite("test_export_basic", "p0") {
PROPERTIES(
"label" = "${label}",
"format" = "csv",
- "column_separator"=","
+ "column_separator"=",",
+ "data_consistency" = "none"
);
"""
waiting_export.call(label_db, label)
diff --git
a/regression-test/suites/export_p0/test_export_data_consistency.groovy
b/regression-test/suites/export_p0/test_export_data_consistency.groovy
index 354010a8582..cd19b082802 100644
--- a/regression-test/suites/export_p0/test_export_data_consistency.groovy
+++ b/regression-test/suites/export_p0/test_export_data_consistency.groovy
@@ -150,8 +150,7 @@ suite("test_export_data_consistency", "p0") {
"label" = "${label}",
"format" = "csv",
"column_separator" = ",",
- "parallelism" = "10",
- "data_consistency" = "partition"
+ "parallelism" = "10"
);
"""
// do insert in parallel
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
