This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7027772afcb [fix](fe) fix Azure resource persistence (#65052)
7027772afcb is described below
commit 7027772afcbf36972662ad0c71dfc9f47bb13f4e
Author: Gavin Chou <[email protected]>
AuthorDate: Thu Jul 2 23:18:39 2026 +0800
[fix](fe) fix Azure resource persistence (#65052)
### What problem does this PR solve?
Issue Number: DORIS-26664
Related PR: None
Problem Summary:
`AzureResource` was not registered in the `Resource` runtime Gson
adapter. As a result, persisted Azure resources could be serialized
without the `clazz` field, and FE replay from edit log or image failed
when deserializing them as `Resource.class`.
Observed failure stack:
```text
2026-06-26 10:16:34,576 ERROR (leaderCheckpointer|100)
[BDBJournalCursor.next():122] fail to read journal entity key=136567, will exit
com.google.gson.JsonParseException: cannot deserialize class
org.apache.doris.catalog.Resource because it does not define a field named clazz
at
org.apache.doris.persist.gson.RuntimeTypeAdapterFactory$1.read(RuntimeTypeAdapterFactory.java:321)
at com.google.gson.TypeAdapter$1.read(TypeAdapter.java:204)
at com.google.gson.Gson.fromJson(Gson.java:1227)
at com.google.gson.Gson.fromJson(Gson.java:1137)
at com.google.gson.Gson.fromJson(Gson.java:1047)
at com.google.gson.Gson.fromJson(Gson.java:982)
at org.apache.doris.catalog.Resource.read(Resource.java:264)
at
org.apache.doris.journal.JournalEntity.readFields(JournalEntity.java:585)
at
org.apache.doris.journal.bdbje.BDBJournalCursor.next(BDBJournalCursor.java:119)
at org.apache.doris.catalog.Env.replayJournal(Env.java:3091)
at
org.apache.doris.master.Checkpoint.doCheckpoint(Checkpoint.java:161)
at
org.apache.doris.master.Checkpoint.runAfterCatalogReady(Checkpoint.java:89)
at
org.apache.doris.common.util.MasterDaemon.runOneCycle(MasterDaemon.java:58)
at org.apache.doris.common.util.Daemon.run(Daemon.java:119)
```
This PR registers `AzureResource` in `GsonUtils` and makes both
`Resource.read` and `ResourceMgr.read` compatible with legacy resource
JSON that has `type` but no `clazz`, deriving the subtype label from
`ResourceType`.
Why the persisted JSON could miss `clazz`:
`clazz` is not a real field of `Resource`; it is the discriminator added
by `RuntimeTypeAdapterFactory` only when that factory handles
serialization. `Resource.write()` currently uses
`GsonUtils.GSON.toJson(this)`. When `this` is an `AzureResource` and
`AzureResource` has not been registered as a `Resource` subtype, Gson
looks up serialization by the runtime type `AzureResource.class`, the
resource runtime type adapter does not match, and Gson falls back to
normal reflective serialization. That reflective path writes real fields
such as `name`, `type`, `references`, `id`, and `version`, but it does
not add `clazz`.
The read path is different: `Resource.read()` deserializes the same JSON
as `Resource.class`, so `RuntimeTypeAdapterFactory` is used and requires
the `clazz` discriminator before it can choose a concrete subtype. The
resources image path also deserializes nested `Resource` values through
`ResourceMgr.read()` and `GsonUtils.GSON.fromJson(json,
ResourceMgr.class)`, so those nested resources do not pass through
`Resource.read()`. Registering `AzureResource` fixes new writes and
reads of JSON that already has `clazz:"AzureResource"`, but it cannot
repair already persisted legacy JSON that has only `type:"AZURE"` and no
`clazz`.
---
.../java/org/apache/doris/catalog/Resource.java | 63 ++++++++++++
.../java/org/apache/doris/catalog/ResourceMgr.java | 27 +++++
.../org/apache/doris/persist/gson/GsonUtils.java | 2 +
.../apache/doris/persist/ResourcePersistTest.java | 110 ++++++++++++++++-----
4 files changed, 180 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
index 96e510c1a83..97aece56b69 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
@@ -33,6 +33,9 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -266,9 +269,69 @@ public abstract class Resource implements Writable,
GsonPostProcessable {
public static Resource read(DataInput in) throws IOException {
String json = Text.readString(in);
+ json = addLegacyClazzIfMissing(json);
return GsonUtils.GSON.fromJson(json, Resource.class);
}
+ // Compatibility for legacy Resource JSON written without
RuntimeTypeAdapterFactory's "clazz"
+ // discriminator. This can be removed after such old metadata no longer
needs to be supported.
+ static String addLegacyClazzIfMissing(String json) {
+ JsonElement jsonElement = JsonParser.parseString(json);
+ if (!jsonElement.isJsonObject()) {
+ return json;
+ }
+
+ JsonObject jsonObject = jsonElement.getAsJsonObject();
+ if (addLegacyClazzIfMissing(jsonObject)) {
+ return jsonObject.toString();
+ }
+ return json;
+ }
+
+ static boolean addLegacyClazzIfMissing(JsonObject jsonObject) {
+ if (jsonObject.has("clazz") || !jsonObject.has("type")) {
+ return false;
+ }
+
+ JsonElement typeElement = jsonObject.get("type");
+ if (!typeElement.isJsonPrimitive()) {
+ return false;
+ }
+
+ ResourceType resourceType =
ResourceType.fromString(typeElement.getAsString());
+ String clazz = getLegacyClazz(resourceType);
+ if (clazz == null) {
+ return false;
+ }
+ jsonObject.addProperty("clazz", clazz);
+ return true;
+ }
+
+ private static String getLegacyClazz(ResourceType resourceType) {
+ switch (resourceType) {
+ case SPARK:
+ return SparkResource.class.getSimpleName();
+ case ODBC_CATALOG:
+ return OdbcCatalogResource.class.getSimpleName();
+ case S3:
+ return S3Resource.class.getSimpleName();
+ case JDBC:
+ return JdbcResource.class.getSimpleName();
+ case HDFS:
+ return HdfsResource.class.getSimpleName();
+ case HMS:
+ return HMSResource.class.getSimpleName();
+ case ES:
+ return EsResource.class.getSimpleName();
+ case AZURE:
+ return AzureResource.class.getSimpleName();
+ case AI:
+ return AIResource.class.getSimpleName();
+ default:
+ return null;
+ }
+ }
+
@Override
public void gsonPostProcess() throws IOException {
// Resource is loaded from meta with older version
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
index 1135a8a35fa..92073f2cb98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
@@ -39,6 +39,9 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -234,9 +237,33 @@ public class ResourceMgr implements Writable {
public static ResourceMgr read(DataInput in) throws IOException {
String json = Text.readString(in);
+ json = addLegacyClazzForResourcesIfMissing(json);
return GsonUtils.GSON.fromJson(json, ResourceMgr.class);
}
+ // ResourceMgr image keeps Resource objects nested in nameToResource, so
Resource.read() is not used.
+ private static String addLegacyClazzForResourcesIfMissing(String json) {
+ JsonElement jsonElement = JsonParser.parseString(json);
+ if (!jsonElement.isJsonObject()) {
+ return json;
+ }
+
+ JsonObject jsonObject = jsonElement.getAsJsonObject();
+ JsonElement resourcesElement = jsonObject.get("nameToResource");
+ if (resourcesElement == null || !resourcesElement.isJsonObject()) {
+ return json;
+ }
+
+ boolean changed = false;
+ for (Map.Entry<String, JsonElement> entry :
resourcesElement.getAsJsonObject().entrySet()) {
+ JsonElement resourceElement = entry.getValue();
+ if (resourceElement != null && resourceElement.isJsonObject()) {
+ changed |=
Resource.addLegacyClazzIfMissing(resourceElement.getAsJsonObject());
+ }
+ }
+ return changed ? jsonObject.toString() : json;
+ }
+
public class ResourceProcNode implements ProcNodeInterface {
@Override
public ProcResult fetchResult() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index ad543a2d0be..d4135f16688 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -75,6 +75,7 @@ import org.apache.doris.catalog.AnyElementType;
import org.apache.doris.catalog.AnyStructType;
import org.apache.doris.catalog.AnyType;
import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.AzureResource;
import org.apache.doris.catalog.BrokerTable;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
@@ -335,6 +336,7 @@ public class GsonUtils {
.registerSubtype(SparkResource.class,
SparkResource.class.getSimpleName())
.registerSubtype(OdbcCatalogResource.class,
OdbcCatalogResource.class.getSimpleName())
.registerSubtype(S3Resource.class,
S3Resource.class.getSimpleName())
+ .registerSubtype(AzureResource.class,
AzureResource.class.getSimpleName())
.registerSubtype(JdbcResource.class,
JdbcResource.class.getSimpleName())
.registerSubtype(HdfsResource.class,
HdfsResource.class.getSimpleName())
.registerSubtype(HMSResource.class,
HMSResource.class.getSimpleName())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
index d69fb57a063..8a542a0897d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
@@ -17,41 +17,107 @@
package org.apache.doris.persist;
+import org.apache.doris.catalog.AzureResource;
import org.apache.doris.catalog.Resource;
+import org.apache.doris.catalog.ResourceMgr;
import org.apache.doris.catalog.S3Resource;
+import org.apache.doris.common.io.Text;
import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
public class ResourcePersistTest {
@Test
public void test() throws IOException {
Resource resource = new S3Resource("s3_resource");
- File file = new File("./ResourcePersistTest");
- try {
- // 1. Write objects to file
- file.createNewFile();
- DataOutputStream dos = new DataOutputStream(new
FileOutputStream(file));
- resource.write(dos);
- dos.flush();
- dos.close();
-
- // 2. Read objects from file
- DataInputStream dis = new DataInputStream(new
FileInputStream(file));
- S3Resource resource1 = (S3Resource) Resource.read(dis);
- dis.close();
- Assert.assertEquals(resource1.toString(), resource.toString());
- resource1.readLock();
- resource1.readUnlock();
- } finally {
- file.delete();
- }
+ S3Resource resource1 = (S3Resource) readWrittenResource(resource);
+ Assert.assertEquals(resource1.toString(), resource.toString());
+ resource1.readLock();
+ resource1.readUnlock();
+ }
+
+ @Test
+ public void testAzureResourcePersist() throws IOException {
+ Resource resource = new AzureResource("azure_resource");
+
Assert.assertTrue(resource.toString().contains("\"clazz\":\"AzureResource\""));
+
+ Resource readResource = readWrittenResource(resource);
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ @Test
+ public void testReadLegacyAzureResourceWithoutClazz() throws IOException {
+ String json = "{\"name\":\"legacy_azure_resource\",\"type\":\"AZURE\","
+ + "\"references\":{},\"id\":123,\"version\":0}";
+
+ Resource readResource = readResourceFromJson(json);
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("legacy_azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ @Test
+ public void testReadLegacyAzureResourceMgrWithoutClazz() throws
IOException {
+ String json =
"{\"nameToResource\":{\"legacy_azure_resource\":{\"name\":\"legacy_azure_resource\","
+ +
"\"type\":\"AZURE\",\"references\":{},\"id\":123,\"version\":0}}}";
+
+ ResourceMgr resourceMgr = readResourceMgrFromJson(json);
+ Resource readResource =
resourceMgr.getResource("legacy_azure_resource");
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("legacy_azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ private Resource readWrittenResource(Resource resource) throws IOException
{
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ resource.write(dos);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ Resource readResource = Resource.read(dis);
+ dis.close();
+ return readResource;
+ }
+
+ private Resource readResourceFromJson(String json) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ Text.writeString(dos, json);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ Resource readResource = Resource.read(dis);
+ dis.close();
+ return readResource;
+ }
+
+ private ResourceMgr readResourceMgrFromJson(String json) throws
IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ Text.writeString(dos, json);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ ResourceMgr resourceMgr = ResourceMgr.read(dis);
+ dis.close();
+ return resourceMgr;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
