This is an automated email from the ASF dual-hosted git repository.
adonisling pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 918141bc7e4 [enhancement](fe-meta) Support skip specified journal to
avoid FE can't be start due to a damaged journal (#35783)
918141bc7e4 is described below
commit 918141bc7e4d98b514f46a736d2e58f5b4875df2
Author: zxealous <[email protected]>
AuthorDate: Wed Jun 5 10:22:18 2024 +0800
[enhancement](fe-meta) Support skip specified journal to avoid FE can't be
start due to a damaged journal (#35783)
If fe meta has damaged journals, fe can't be started successfully.
Adding a configuration to skip the damaged journals.
---
.../main/java/org/apache/doris/common/Config.java | 6 ++++++
.../src/main/java/org/apache/doris/catalog/Env.java | 21 ++++++++++++++++++++-
.../doris/journal/bdbje/BDBJournalCursor.java | 4 ++++
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 94ad7ea7b36..a266da356a6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1223,6 +1223,12 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean force_do_metadata_checkpoint = false;
+ /**
+ * If some joural is wrong, and FE can't start, we can use this to skip it.
+ */
+ @ConfField(mutable = false, masterOnly = false)
+ public static String[] force_skip_journal_ids = {};
+
/**
* Decide how often to check dynamic partition
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index e5f0dcaf585..f6db594d770 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -307,6 +307,7 @@ import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
@@ -551,6 +552,8 @@ public class Env {
private final SplitSourceManager splitSourceManager;
+ private final List<String> forceSkipJournalIds =
Arrays.asList(Config.force_skip_journal_ids);
+
public List<TFrontendInfo> getFrontendInfos() {
List<TFrontendInfo> res = new ArrayList<>();
@@ -963,6 +966,10 @@ public class Env {
return dnsCache;
}
+ public List<String> getForceSkipJournalIds() {
+ return forceSkipJournalIds;
+ }
+
// Use tryLock to avoid potential dead lock
private boolean tryLock(boolean mustLock) {
while (true) {
@@ -2868,7 +2875,19 @@ public class Env {
Long logId = kv.first;
JournalEntity entity = kv.second;
if (entity == null) {
- break;
+ if (logId != null &&
forceSkipJournalIds.contains(String.valueOf(logId))) {
+ replayedJournalId.incrementAndGet();
+ String msg = "journal " + replayedJournalId + " has
skipped by config force_skip_journal_id";
+ LOG.info(msg);
+ LogUtils.stdout(msg);
+ if (MetricRepo.isInit) {
+ // Metric repo may not init after this replay thread
start
+ MetricRepo.COUNTER_EDIT_LOG_READ.increase(1L);
+ }
+ continue;
+ } else {
+ break;
+ }
}
hasLog = true;
EditLog.loadJournal(this, logId, entity);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java
index 8c48cab42c5..f6068d5948b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java
@@ -17,6 +17,7 @@
package org.apache.doris.journal.bdbje;
+import org.apache.doris.catalog.Env;
import org.apache.doris.common.Pair;
import org.apache.doris.journal.JournalCursor;
import org.apache.doris.journal.JournalEntity;
@@ -93,6 +94,9 @@ public class BDBJournalCursor implements JournalCursor {
return null;
}
+ if
(Env.getCurrentEnv().getForceSkipJournalIds().contains(String.valueOf(currentKey)))
{
+ return Pair.of(currentKey++, null);
+ }
Long key = currentKey;
DatabaseEntry theKey = new DatabaseEntry();
TupleBinding<Long> myBinding =
TupleBinding.getPrimitiveBinding(Long.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
