This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new d085c5ea715 [fix](fe) Add check editlog size mechanism for backupJob
(#35653) (#37628)
d085c5ea715 is described below
commit d085c5ea7155a3911e43113a172663c917072f8b
Author: Lei Zhang <[email protected]>
AuthorDate: Fri Jul 12 15:49:45 2024 +0800
[fix](fe) Add check editlog size mechanism for backupJob (#35653) (#37628)
* When creating a backupJob with huge of tables in a database, it can
cause backupJob editlog size over 2GB and bdbje will throw exception
because of ByteBuffer overflow
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
.../java/org/apache/doris/backup/BackupJob.java | 7 +++++++
.../java/org/apache/doris/journal/Journal.java | 2 ++
.../apache/doris/journal/bdbje/BDBJEJournal.java | 23 ++++++++++++++++++++++
.../apache/doris/journal/local/LocalJournal.java | 5 +++++
.../java/org/apache/doris/persist/EditLog.java | 13 ++++++++++++
5 files changed, 50 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index b6f031ab7d4..ec22d6eafc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -568,6 +568,13 @@ public class BackupJob extends AbstractJob {
private void waitingAllSnapshotsFinished() {
if (unfinishedTaskIds.isEmpty()) {
+
+ if (env.getEditLog().exceedMaxJournalSize(this)) {
+ status = new Status(ErrCode.COMMON_ERROR, "backupJob is too
large ");
+ return;
+ }
+
+
snapshotFinishedTime = System.currentTimeMillis();
state = BackupJobState.UPLOAD_SNAPSHOT;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java
index 8fca299df19..b0929818eed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java
@@ -61,4 +61,6 @@ public interface Journal {
// Get all the dbs' name
public List<Long> getDatabaseNames();
+ public boolean exceedMaxJournalSize(short op, Writable writable) throws
IOException;
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
index 85e577058be..3c4662e2017 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
@@ -586,4 +586,27 @@ public class BDBJEJournal implements Journal { //
CHECKSTYLE IGNORE THIS LINE: B
public BDBEnvironment getBDBEnvironment() {
return this.bdbEnvironment;
}
+
+ @Override
+ public boolean exceedMaxJournalSize(short op, Writable writable) throws
IOException {
+ JournalEntity entity = new JournalEntity();
+ entity.setOpCode(op);
+ entity.setData(writable);
+
+ DataOutputBuffer buffer = new
DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE);
+ entity.write(buffer);
+
+ DatabaseEntry theData = new DatabaseEntry(buffer.getData());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("opCode = {}, journal size = {}", op, theData.getSize());
+ }
+
+ // 1GB
+ if (theData.getSize() > (1 << 30)) {
+ return true;
+ }
+
+ return false;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java
index 9ba3274e0f0..4467c53a368 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java
@@ -198,4 +198,9 @@ public class LocalJournal implements Journal {
public List<Long> getDatabaseNames() {
throw new RuntimeException("Not Support");
}
+
+ @Override
+ public boolean exceedMaxJournalSize(short op, Writable writable) throws
IOException {
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 79dde3e5874..a1e033d6d1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -1905,4 +1905,17 @@ public class EditLog {
public void logDeleteTableStats(TableStatsDeletionLog log) {
logEdit(OperationType.OP_DELETE_TABLE_STATS, log);
}
+
+ public boolean exceedMaxJournalSize(BackupJob job) {
+ try {
+ return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job);
+ } catch (Exception e) {
+ LOG.warn("exceedMaxJournalSize exception:", e);
+ }
+ return true;
+ }
+
+ private boolean exceedMaxJournalSize(short op, Writable writable) throws
IOException {
+ return journal.exceedMaxJournalSize(op, writable);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
