This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e4d543f0605 [branch-2.1](load) fix commit txn timeout when loading to
table with many tablet (#40031) (#40139)
e4d543f0605 is described below
commit e4d543f06050af37b1b9ada7773bbd639030bfa9
Author: zclllhhjj <[email protected]>
AuthorDate: Fri Aug 30 10:34:43 2024 +0800
[branch-2.1](load) fix commit txn timeout when loading to table with many
tablet (#40031) (#40139)
pick https://github.com/apache/doris/pull/40031
---
.../org/apache/doris/load/BrokerFileGroup.java | 16 +++++----
.../doris/transaction/DatabaseTransactionMgr.java | 40 +++++++++++++++-------
2 files changed, 38 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
index ff27acadcac..6676e0a2a1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
@@ -149,12 +149,16 @@ public class BrokerFileGroup implements Writable {
}
}
- boolean isPartitionRestoring =
olapTable.getPartitions().stream().anyMatch(
- partition -> partition.getState() == PartitionState.RESTORE
- );
- // restore table
- if (!isPartitionRestoring && olapTable.getState() ==
OlapTableState.RESTORE) {
- throw new DdlException("Table [" + olapTable.getName() + "] is
under restore");
+ // only do check when here's restore on this table now
+ if (olapTable.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring =
olapTable.getPartitions().stream()
+ .anyMatch(partition -> partition.getState() ==
PartitionState.RESTORE);
+ // tbl RESTORE && all partition NOT RESTORE -> whole table
restore
+ // tbl RESTORE && some partition RESTORE -> just partitions
restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new DdlException("Table [" + olapTable.getName() +
"] is under restore");
+ }
}
if (olapTable.getKeysType() != KeysType.AGG_KEYS &&
dataDescription.isNegative()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 636fe582ed1..38514aff8d6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -504,7 +504,9 @@ public class DatabaseTransactionMgr {
List<Long> tabletIds = tabletCommitInfos.stream()
.map(TabletCommitInfo::getTabletId).collect(Collectors.toList());
List<TabletMeta> tabletMetaList =
tabletInvertedIndex.getTabletMetaList(tabletIds);
+ HashMap<Long, Boolean> tableIdtoRestoring = new HashMap<>();
for (int i = 0; i < tabletMetaList.size(); i++) {
+ // get partition and table of this tablet
TabletMeta tabletMeta = tabletMetaList.get(i);
if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
continue;
@@ -513,29 +515,43 @@ public class DatabaseTransactionMgr {
long tableId = tabletMeta.getTableId();
OlapTable tbl = (OlapTable) idToTable.get(tableId);
if (tbl == null) {
- // this can happen when tableId == -1 (tablet being dropping)
- // or table really not exist.
+ // this can happen when tableId == -1 (tablet being dropping)
or table really not exist.
continue;
}
+ // check relative partition restore here
long partitionId = tabletMeta.getPartitionId();
if (tbl.getPartition(partitionId) == null) {
- // this can happen when partitionId == -1 (tablet being
dropping)
- // or partition really not exist.
+ // this can happen when partitionId == -1 (tablet being
dropping) or partition really not exist.
continue;
- } else if (tbl.getPartition(partitionId).getState() ==
PartitionState.RESTORE) {
+ }
+ if (tbl.getPartition(partitionId).getState() ==
PartitionState.RESTORE) {
// partition in restore process which can not load data
throw new LoadException("Table [" + tbl.getName() + "],
Partition ["
+ tbl.getPartition(partitionId).getName() + "] is in
restore process. Can not load into it");
}
- boolean isPartitionRestoring =
tbl.getPartitions().stream().anyMatch(
- partition -> partition.getState() == PartitionState.RESTORE
- );
- // restore table
- if (!isPartitionRestoring && tbl.getState() ==
OlapTableState.RESTORE) {
- throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
- + "Can not load into it");
+ // only do check when here's restore on this table now
+ if (tbl.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring = false;
+ if (tableIdtoRestoring.containsKey(tableId)) {
+ hasPartitionRestoring = tableIdtoRestoring.get(tableId);
+ } else {
+ for (Partition partition : tbl.getPartitions()) {
+ if (partition.getState() == PartitionState.RESTORE) {
+ hasPartitionRestoring = true;
+ break;
+ }
+ }
+ tableIdtoRestoring.put(tableId, hasPartitionRestoring);
+ }
+ // tbl RESTORE && all partition NOT RESTORE -> whole table
restore
+ // tbl RESTORE && some partition RESTORE -> just partitions
restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new LoadException(
+ "Table " + tbl.getName() + " is in restore
process. " + "Can not load into it");
+ }
}
if (!tableToPartition.containsKey(tableId)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
