This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new ab4d81a6761 [enhancement](Load)allow load data to the other partitions
when some partitions are restoring (#39411)
ab4d81a6761 is described below
commit ab4d81a6761a6dea7bb6634bca2ae1f99c192bdb
Author: shouchengShen <[email protected]>
AuthorDate: Mon Aug 26 15:22:47 2024 +0800
[enhancement](Load)allow load data to the other partitions when some
partitions are restoring (#39411)
If broker load or stream load task execute in one table that is
restoring data, load task will failed with Exception.
Exception info :"Table [xxx] is under restore" or "Table [xxx] is in
restore process, can't load into it".
But mostly restoreJob only effects some partitions in this table, not
all of them, so that the other partitions still need to load data
successfully.
To achieve this goal, before checking olap table state, check partition
state first.
ps: set restore status for partitions in this
pr:https://github.com/apache/doris/pull/8245
## test case for this pr
### restore tbl's partition p202408
$ RESTORE SNAPSHOT db.tbl_p202408_test
FROM repo
ON(
`tbl` PARTITION (p202408)
)
PROPERTIES(
"backup_timestamp"="2024-08-22-20-32-37",
"replication_num" = "1"
);
### check restore job state\G
$ SHOW RESTORE\G
*************************** 1. row ***************************
JobId: 21741
Label: tbl_p202408_test
Timestamp: 2024-08-22-20-32-37
State: DOWNLOADING
RestoreObjs: {
"name": "tbl_p202408_test",
"database": "db",
"olap_table_list": [
{
"name": "tbl",
"partition_names": ["p202408"]
}
]
### load to partition p202408, failed with exception
curl --location-trusted -u root:"" \
> -H "label:tbl_test_load_19" \
> -H "timeout:300" \
> -H "format: parquet" \
> -T data_for_p202408.parquet \
> -XPUT http://fe_ip:8030/api/db/tbl/_stream_load
{
"TxnId": 3042,
"Label": "tbl_test_load_19",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = Table
[zt_order_detail_v3], Partition [p202408] is in restore process. Can not
load into it.etc.",
"NumberTotalRows": 682,
"NumberLoadedRows": 682,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 82025,
"LoadTimeMs": 48,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 7,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 38,
"CommitAndPublishTimeMs": 0
}
### load to partition p202408, successfully
$ curl --location-trusted -u root:"" \
> -H "timeout:300" \
> -H "format: json" \
> -H "read_json_by_line:true" \
> -T data_for_p202407.json \
> -XPUT http://fe_ip:8030/api/db/tbl/_stream_load
{
"TxnId": 3043,
"Label": "2f2dae38-a495-4c22-9492-419ea70b724e",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Success",
"Message": "OK",
"NumberTotalRows": 1,
"NumberLoadedRows": 1,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 1128,
"LoadTimeMs": 51,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 6,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 30,
"CommitAndPublishTimeMs": 13
}
Co-authored-by: shenshoucheng <[email protected]>
---
.../org/apache/doris/load/BrokerFileGroup.java | 13 ++++++++++++-
.../doris/transaction/DatabaseTransactionMgr.java | 22 ++++++++++++++++------
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
index 2b493eefb45..35bd0b8db7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
@@ -31,6 +31,7 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
@@ -181,11 +182,21 @@ public class BrokerFileGroup implements Writable {
throw new DdlException("Unknown partition '" + pName
+ "' in table '" + olapTable.getName() + "'");
}
+ // partition which need load data
+ if (partition.getState() == PartitionState.RESTORE) {
+ throw new DdlException("Table [" + olapTable.getName()
+ + "], Partition[" + partition.getName() + "]
is under restore");
+ }
partitionIds.add(partition.getId());
}
}
- if (olapTable.getState() == OlapTableState.RESTORE) {
+ boolean isPartitionRestoring =
olapTable.getPartitions().stream().anyMatch(
+ partition -> partition.getState() == PartitionState.RESTORE
+ );
+
+ // restore table
+ if (!isPartitionRestoring && olapTable.getState() ==
OlapTableState.RESTORE) {
throw new DdlException("Table [" + olapTable.getName() + "] is
under restore");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index d733d863f4e..d99f9ba60f6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -22,7 +22,9 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.Replica;
@@ -474,16 +476,24 @@ public class DatabaseTransactionMgr {
continue;
}
- if (tbl.getState() == OlapTable.OlapTableState.RESTORE) {
- throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
- + "Can not load into it");
- }
-
long partitionId = tabletMeta.getPartitionId();
- if (tbl.getPartition(partitionId) == null) {
+ Partition partition = tbl.getPartition(partitionId);
+ if (partition == null) {
// this can happen when partitionId == -1 (tablet being
dropping)
// or partition really not exist.
continue;
+ } else if (partition.getState() == PartitionState.RESTORE) {
+ // partition which need load data
+ throw new LoadException("Table [" + tbl.getName() + "],
Partition ["
+ + partition.getName() + "] is in restore process. Can
not load into it");
+ }
+ boolean isPartitionRestoring =
tbl.getPartitions().stream().anyMatch(
+ curPartition -> curPartition.getState() ==
PartitionState.RESTORE
+ );
+ // restore table
+ if (!isPartitionRestoring && tbl.getState() ==
OlapTableState.RESTORE) {
+ throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
+ + "Can not load into it");
}
if (!tableToPartition.containsKey(tableId)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
