This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new be398bc258 [fix](catalog) fix NPE when replay from FE image (#18499)
be398bc258 is described below
commit be398bc25886521afef8206a5db937da5b9ba04d
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon Apr 10 09:26:03 2023 +0800
[fix](catalog) fix NPE when replay from FE image (#18499)
Introduced from #17884.
When replay catalog from image, we should not call
`catalog.getProperties()`.
Because it will visit the resource mgr, but resource mgr is not replayed
yet.
---
.../src/main/java/org/apache/doris/catalog/Env.java | 12 ++++++++----
.../java/org/apache/doris/datasource/CatalogMgr.java | 18 ++++++++++++------
.../main/java/org/apache/doris/master/Checkpoint.java | 2 +-
3 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index af1aa6b4bf..a68c2e573a 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1317,7 +1317,7 @@ public class Env {
// for master, the 'isReady' is set behind.
// but we are sure that all metadata is replayed if we get here.
// so no need to check 'isReady' flag in this method
- fixBugAfterMetadataReplayed(false);
+ postProcessAfterMetadataReplayed(false);
// start all daemon threads that only running on MASTER FE
startMasterOnlyDaemonThreads();
@@ -1338,9 +1338,12 @@ public class Env {
}
/*
- * Add anything necessary here if there is meta data need to be fixed.
+ * There are something need to do after metadata is replayed, such as
+ * 1. bug fix for metadata
+ * 2. register some hook.
+ * If there is, add them here.
*/
- public void fixBugAfterMetadataReplayed(boolean waitCatalogReady) {
+ public void postProcessAfterMetadataReplayed(boolean waitCatalogReady) {
if (waitCatalogReady) {
while (!isReady()) {
try {
@@ -1352,6 +1355,7 @@ public class Env {
}
auth.rectifyPrivs();
+ catalogMgr.registerCatalogRefreshListener(this);
}
// start all daemon threads only running on Master
@@ -1464,7 +1468,7 @@ public class Env {
}
// 'isReady' will be set to true in 'setCanRead()' method
- fixBugAfterMetadataReplayed(true);
+ postProcessAfterMetadataReplayed(true);
checkLowerCaseTableNames();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index a116f0a64e..1d219da65f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -963,6 +963,17 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
log.getPartitionNames());
}
+ public void registerCatalogRefreshListener(Env env) {
+ for (CatalogIf catalog : idToCatalog.values()) {
+ Map<String, String> properties = catalog.getProperties();
+ if (properties.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
+ Integer metadataRefreshIntervalSec =
Integer.valueOf(properties.get(METADATA_REFRESH_INTERVAL_SEC));
+ Integer[] sec = {metadataRefreshIntervalSec,
metadataRefreshIntervalSec};
+ env.getRefreshManager().addToRefreshMap(catalog.getId(), sec);
+ }
+ }
+ }
+
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this);
@@ -973,12 +984,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
public void gsonPostProcess() throws IOException {
for (CatalogIf catalog : idToCatalog.values()) {
nameToCatalog.put(catalog.getName(), catalog);
- Map properties = catalog.getProperties();
- if (properties.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
- Integer metadataRefreshIntervalSec = (Integer)
properties.get(METADATA_REFRESH_INTERVAL_SEC);
- Integer[] sec = {metadataRefreshIntervalSec,
metadataRefreshIntervalSec};
-
Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalog.getId(), sec);
- }
+ // ATTN: can not call catalog.getProperties() here, because
ResourceMgr is not replayed yet.
}
internalCatalog = (InternalCatalog)
idToCatalog.get(InternalCatalog.INTERNAL_CATALOG_ID);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
index 36f8a53938..011d40b8a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
@@ -128,7 +128,7 @@ public class Checkpoint extends MasterDaemon {
String.format("checkpoint version should be %d," + "
actual replayed journal id is %d",
checkPointVersion,
env.getReplayedJournalId()));
}
- env.fixBugAfterMetadataReplayed(false);
+ env.postProcessAfterMetadataReplayed(false);
latestImageFilePath = env.saveImage();
replayedJournalId = env.getReplayedJournalId();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
