This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f18d7b6c728 [fix](group commit) Fix group commit debug log and improve
performance (#38754)
f18d7b6c728 is described below
commit f18d7b6c72866b29b9639bcfccf279a8742f5304
Author: meiyi <[email protected]>
AuthorDate: Fri Aug 2 20:07:38 2024 +0800
[fix](group commit) Fix group commit debug log and improve performance
(#38754)
## Proposed changes
1. show `query_id` in the debug log of when group commit insert does not
work
2. remove string.format to improve performance
---
.../org/apache/doris/analysis/NativeInsertStmt.java | 20 ++++++++++++--------
.../insert/OlapGroupCommitInsertExecutor.java | 7 ++++---
.../main/java/org/apache/doris/qe/StmtExecutor.java | 6 +++---
.../insert_p0/txn_insert_concurrent_insert.groovy | 2 +-
4 files changed, 20 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index bc2ee01c58f..ae3c2371e73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -1242,8 +1242,9 @@ public class NativeInsertStmt extends InsertStmt {
for (Expr expr : row) {
if (!(expr instanceof LiteralExpr)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("group commit is off for table: {},
because not literal expr, "
- + "expr: {}, row: {}",
targetTable.getName(), expr, row);
+ LOG.debug("group commit is off for query_id:
{}, table: {}, "
+ + "because not literal expr:
{}, row: {}",
+ DebugUtil.printId(ctx.queryId()),
targetTable.getName(), expr, row);
}
return;
}
@@ -1252,8 +1253,9 @@ public class NativeInsertStmt extends InsertStmt {
// Does not support: insert into tbl values();
if (selectStmt.getValueList().getFirstRow().isEmpty() &&
CollectionUtils.isEmpty(targetColumnNames)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("group commit is off for table: {}, because
first row: {}, target columns: {}",
- targetTable.getName(),
selectStmt.getValueList().getFirstRow(), targetColumnNames);
+ LOG.debug("group commit is off for query_id: {},
table: {}, because first row: {}, "
+ + "target columns: {}",
DebugUtil.printId(ctx.queryId()), targetTable.getName(),
+ selectStmt.getValueList().getFirstRow(),
targetColumnNames);
}
return;
}
@@ -1265,8 +1267,10 @@ public class NativeInsertStmt extends InsertStmt {
for (SelectListItem item : items) {
if (item.getExpr() != null && !(item.getExpr()
instanceof LiteralExpr)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("group commit is off for table:
{}, because not literal expr, "
- + "expr: {}, row: {}",
targetTable.getName(), item.getExpr(), item);
+ LOG.debug("group commit is off for
query_id: {}, table: {}, "
+ + "because not literal
expr: {}, row: {}",
+ DebugUtil.printId(ctx.queryId()),
targetTable.getName(), item.getExpr(),
+ item);
}
return;
}
@@ -1279,8 +1283,8 @@ public class NativeInsertStmt extends InsertStmt {
if (LOG.isDebugEnabled()) {
for (Pair<BooleanSupplier, Supplier<String>> pair :
conditions) {
if (pair.first.getAsBoolean() == false) {
- LOG.debug("group commit is off for table: {}, because:
{}", targetTable.getName(),
- pair.second.get());
+ LOG.debug("group commit is off for query_id: {},
table: {}, because: {}",
+ DebugUtil.printId(ctx.queryId()),
targetTable.getName(), pair.second.get());
break;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
index 496bc161a5c..a44528e93b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
@@ -97,15 +97,16 @@ public class OlapGroupCommitInsertExecutor extends
OlapInsertExecutor {
if (!ctx.isGroupCommit() && LOG.isDebugEnabled()) {
for (Pair<BooleanSupplier, Supplier<String>> pair :
conditions) {
if (pair.first.getAsBoolean() == false) {
- LOG.debug("group commit is off for table: {}, because:
{}", table.getName(), pair.second.get());
+ LOG.debug("group commit is off for query_id: {},
table: {}, because: {}",
+ DebugUtil.printId(ctx.queryId()),
table.getName(), pair.second.get());
break;
}
}
}
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug("group commit is off because for table: {}, because
logicalQuery class: {}", table.getName(),
- logicalQuery.getClass().getName());
+ LOG.debug("group commit is off for query_id: {}, table: {},
because logicalQuery class: {}",
+ DebugUtil.printId(ctx.queryId()), table.getName(),
logicalQuery.getClass().getName());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index adb8ecf61bf..e39fb1557ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -366,14 +366,14 @@ public class StmtExecutor {
if (expr instanceof NullLiteral) {
row.addColBuilder().setValue(NULL_VALUE_FOR_LOAD);
} else if (expr instanceof ArrayLiteral) {
- row.addColBuilder().setValue(String.format("\"%s\"",
expr.getStringValueForStreamLoad(options)));
+ row.addColBuilder().setValue("\"" +
expr.getStringValueForStreamLoad(options) + "\"");
} else {
String stringValue = expr.getStringValueForStreamLoad(options);
if (stringValue.equals(NULL_VALUE_FOR_LOAD) ||
stringValue.startsWith("\"") || stringValue.endsWith(
"\"")) {
- row.addColBuilder().setValue(String.format("\"%s\"",
stringValue));
+ row.addColBuilder().setValue("\"" + stringValue + "\"");
} else {
- row.addColBuilder().setValue(String.format("%s",
stringValue));
+ row.addColBuilder().setValue(stringValue);
}
}
}
diff --git
a/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy
b/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy
index c28f9192258..e520d0b9d95 100644
--- a/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy
+++ b/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy
@@ -109,7 +109,7 @@ suite("txn_insert_concurrent_insert") {
futures.add(future)
}
CompletableFuture<?>[] futuresArray = futures.toArray(new
CompletableFuture[0])
- CompletableFuture.allOf(futuresArray).get(3, TimeUnit.MINUTES)
+ CompletableFuture.allOf(futuresArray).get(10, TimeUnit.MINUTES)
sql """ sync """
def result = sql """ select count() from ${tableName}_0 """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
