This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new d46ac0239e0 [Fix](Export) Fix the bug that `tablets_num` is always
zero in `show export` result (#30125)
d46ac0239e0 is described below
commit d46ac0239e0ec0d58f552784accda66e32266815
Author: Tiewei Fang <[email protected]>
AuthorDate: Mon Jan 22 11:17:29 2024 +0800
[Fix](Export) Fix the bug that `tablets_num` is always zero in `show
export` result (#30125)
Bug description:
when we exec `show export`, the `tablets_num` is always zero:
```sql
*************************** 7. row ***************************
JobId: 58303
Label: export_672f66ec-592e-4cd6-8329-13f5a89d93b7
State: FINISHED
Progress: 100%
TaskInfo:
{"partitions":["*"],"max_file_size":"256MB","delete_existing_files":"","columns":"","format":"orc","broker":"HDFS","column_separator":"\t","line_delimiter":"\n","db":"default_cluster:testLi","tbl":"customer","tablet_num":0}
Path: hdfs://xxxx/
CreateTime: 2024-01-09 11:41:02
StartTime: 2024-01-09 11:41:32
FinishTime: 2024-01-09 11:43:03
Timeout: 3600
ErrorMsg: NULL
OutfileInfo: [
{
"fileNumber": "7",
"totalRows": "30000000",
"fileSize": "1669449988bytes",
"url": "hdfs://xxxx"
}
]
```
---
.../src/main/java/org/apache/doris/load/ExportJob.java | 17 +++++++++--------
.../src/main/java/org/apache/doris/load/ExportMgr.java | 2 +-
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 0585311a1ab..03f33bea30f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -58,7 +58,6 @@ import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.task.ExportExportingTask;
import org.apache.doris.thrift.TNetworkAddress;
-import org.apache.doris.thrift.TScanRangeLocations;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
@@ -150,6 +149,8 @@ public class ExportJob implements Writable {
private ExportFailMsg failMsg;
@SerializedName("outfileInfo")
private String outfileInfo;
+ @SerializedName("tabletsNum")
+ private Integer tabletsNum;
// progress has two functions at EXPORTING stage:
// 1. when progress < 100, it indicates exporting
// 2. set progress = 100 ONLY when exporting progress is completely done
@@ -186,7 +187,6 @@ public class ExportJob implements Writable {
private ExportExportingTask task;
- private List<TScanRangeLocations> tabletLocations = Lists.newArrayList();
// backend_address => snapshot path
private List<Pair<TNetworkAddress, String>> snapshotPaths =
Lists.newArrayList();
@@ -341,6 +341,7 @@ public class ExportJob implements Writable {
}
Integer tabletsAllNum = tabletIdList.size();
+ this.tabletsNum = tabletsAllNum;
Integer tabletsNumPerQuery = tabletsAllNum / this.parallelNum;
Integer tabletsNumPerQueryRemainder = tabletsAllNum -
tabletsNumPerQuery * this.parallelNum;
@@ -355,13 +356,13 @@ public class ExportJob implements Writable {
+ "set parallelism to tablets num.", id,
tabletsAllNum, this.parallelNum);
}
for (int i = 0; i < outfileNum; ++i) {
- Integer tabletsNum = tabletsNumPerQuery;
+ Integer realTabletsNum = tabletsNumPerQuery;
if (tabletsNumPerQueryRemainder > 0) {
- tabletsNum = tabletsNum + 1;
+ realTabletsNum = realTabletsNum + 1;
--tabletsNumPerQueryRemainder;
}
- ArrayList<Long> tablets = new
ArrayList<>(tabletIdList.subList(start, start + tabletsNum));
- start += tabletsNum;
+ ArrayList<Long> tablets = new
ArrayList<>(tabletIdList.subList(start, start + realTabletsNum));
+ start += realTabletsNum;
// Since export does not support the alias, here we pass the null
value.
// we can not use this.tableRef.getAlias(),
// because the constructor of `Tableref` will convert
this.tableRef.getAlias()
@@ -547,8 +548,8 @@ public class ExportJob implements Writable {
return this.stmtExecutorList.get(idx);
}
- public List<TScanRangeLocations> getTabletLocations() {
- return tabletLocations;
+ public Integer getTabletsNum() {
+ return this.tabletsNum;
}
public List<Pair<TNetworkAddress, String>> getSnapshotPaths() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 2779fdeadbc..06960c68ff2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -422,7 +422,7 @@ public class ExportMgr extends MasterDaemon {
infoMap.put("format", job.getFormat());
infoMap.put("line_delimiter", job.getLineDelimiter());
infoMap.put("columns", job.getColumns());
- infoMap.put("tablet_num", job.getTabletLocations() == null ? -1 :
job.getTabletLocations().size());
+ infoMap.put("tablet_num", job.getTabletsNum());
infoMap.put("max_file_size", job.getMaxFileSize());
infoMap.put("delete_existing_files", job.getDeleteExistingFiles());
jobInfo.add(new Gson().toJson(infoMap));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
