This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8ef0476c437 branch-4.1: [fix](fe) fix Azure resource persistence
#65052 (#65177)
8ef0476c437 is described below
commit 8ef0476c43754a873c078383f9614e28190aa871
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jul 3 20:53:12 2026 +0800
branch-4.1: [fix](fe) fix Azure resource persistence #65052 (#65177)
Cherry-picked from #65052
Co-authored-by: Gavin Chou <[email protected]>
---
.../java/org/apache/doris/catalog/Resource.java | 63 ++++++++++++
.../java/org/apache/doris/catalog/ResourceMgr.java | 27 +++++
.../org/apache/doris/persist/gson/GsonUtils.java | 2 +
.../apache/doris/persist/ResourcePersistTest.java | 110 ++++++++++++++++-----
4 files changed, 180 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
index db0659c4b08..8250cbca562 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
@@ -32,6 +32,9 @@ import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -266,9 +269,69 @@ public abstract class Resource implements Writable,
GsonPostProcessable {
public static Resource read(DataInput in) throws IOException {
String json = Text.readString(in);
+ json = addLegacyClazzIfMissing(json);
return GsonUtils.GSON.fromJson(json, Resource.class);
}
+ // Compatibility for legacy Resource JSON written without
RuntimeTypeAdapterFactory's "clazz"
+ // discriminator. This can be removed after such old metadata no longer
needs to be supported.
+ static String addLegacyClazzIfMissing(String json) {
+ JsonElement jsonElement = JsonParser.parseString(json);
+ if (!jsonElement.isJsonObject()) {
+ return json;
+ }
+
+ JsonObject jsonObject = jsonElement.getAsJsonObject();
+ if (addLegacyClazzIfMissing(jsonObject)) {
+ return jsonObject.toString();
+ }
+ return json;
+ }
+
+ static boolean addLegacyClazzIfMissing(JsonObject jsonObject) {
+ if (jsonObject.has("clazz") || !jsonObject.has("type")) {
+ return false;
+ }
+
+ JsonElement typeElement = jsonObject.get("type");
+ if (!typeElement.isJsonPrimitive()) {
+ return false;
+ }
+
+ ResourceType resourceType =
ResourceType.fromString(typeElement.getAsString());
+ String clazz = getLegacyClazz(resourceType);
+ if (clazz == null) {
+ return false;
+ }
+ jsonObject.addProperty("clazz", clazz);
+ return true;
+ }
+
+ private static String getLegacyClazz(ResourceType resourceType) {
+ switch (resourceType) {
+ case SPARK:
+ return SparkResource.class.getSimpleName();
+ case ODBC_CATALOG:
+ return OdbcCatalogResource.class.getSimpleName();
+ case S3:
+ return S3Resource.class.getSimpleName();
+ case JDBC:
+ return JdbcResource.class.getSimpleName();
+ case HDFS:
+ return HdfsResource.class.getSimpleName();
+ case HMS:
+ return HMSResource.class.getSimpleName();
+ case ES:
+ return EsResource.class.getSimpleName();
+ case AZURE:
+ return AzureResource.class.getSimpleName();
+ case AI:
+ return AIResource.class.getSimpleName();
+ default:
+ return null;
+ }
+ }
+
@Override
public void gsonPostProcess() throws IOException {
// Resource is loaded from meta with older version
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
index 1135a8a35fa..92073f2cb98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
@@ -39,6 +39,9 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -234,9 +237,33 @@ public class ResourceMgr implements Writable {
public static ResourceMgr read(DataInput in) throws IOException {
String json = Text.readString(in);
+ json = addLegacyClazzForResourcesIfMissing(json);
return GsonUtils.GSON.fromJson(json, ResourceMgr.class);
}
+ // ResourceMgr image keeps Resource objects nested in nameToResource, so
Resource.read() is not used.
+ private static String addLegacyClazzForResourcesIfMissing(String json) {
+ JsonElement jsonElement = JsonParser.parseString(json);
+ if (!jsonElement.isJsonObject()) {
+ return json;
+ }
+
+ JsonObject jsonObject = jsonElement.getAsJsonObject();
+ JsonElement resourcesElement = jsonObject.get("nameToResource");
+ if (resourcesElement == null || !resourcesElement.isJsonObject()) {
+ return json;
+ }
+
+ boolean changed = false;
+ for (Map.Entry<String, JsonElement> entry :
resourcesElement.getAsJsonObject().entrySet()) {
+ JsonElement resourceElement = entry.getValue();
+ if (resourceElement != null && resourceElement.isJsonObject()) {
+ changed |=
Resource.addLegacyClazzIfMissing(resourceElement.getAsJsonObject());
+ }
+ }
+ return changed ? jsonObject.toString() : json;
+ }
+
public class ResourceProcNode implements ProcNodeInterface {
@Override
public ProcResult fetchResult() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index 502df58765f..1097a31232a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -71,6 +71,7 @@ import org.apache.doris.catalog.AnyElementType;
import org.apache.doris.catalog.AnyStructType;
import org.apache.doris.catalog.AnyType;
import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.AzureResource;
import org.apache.doris.catalog.BrokerTable;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
@@ -365,6 +366,7 @@ public class GsonUtils {
.registerSubtype(SparkResource.class,
SparkResource.class.getSimpleName())
.registerSubtype(OdbcCatalogResource.class,
OdbcCatalogResource.class.getSimpleName())
.registerSubtype(S3Resource.class,
S3Resource.class.getSimpleName())
+ .registerSubtype(AzureResource.class,
AzureResource.class.getSimpleName())
.registerSubtype(JdbcResource.class,
JdbcResource.class.getSimpleName())
.registerSubtype(HdfsResource.class,
HdfsResource.class.getSimpleName())
.registerSubtype(HMSResource.class,
HMSResource.class.getSimpleName())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
index d69fb57a063..8a542a0897d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java
@@ -17,41 +17,107 @@
package org.apache.doris.persist;
+import org.apache.doris.catalog.AzureResource;
import org.apache.doris.catalog.Resource;
+import org.apache.doris.catalog.ResourceMgr;
import org.apache.doris.catalog.S3Resource;
+import org.apache.doris.common.io.Text;
import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
public class ResourcePersistTest {
@Test
public void test() throws IOException {
Resource resource = new S3Resource("s3_resource");
- File file = new File("./ResourcePersistTest");
- try {
- // 1. Write objects to file
- file.createNewFile();
- DataOutputStream dos = new DataOutputStream(new
FileOutputStream(file));
- resource.write(dos);
- dos.flush();
- dos.close();
-
- // 2. Read objects from file
- DataInputStream dis = new DataInputStream(new
FileInputStream(file));
- S3Resource resource1 = (S3Resource) Resource.read(dis);
- dis.close();
- Assert.assertEquals(resource1.toString(), resource.toString());
- resource1.readLock();
- resource1.readUnlock();
- } finally {
- file.delete();
- }
+ S3Resource resource1 = (S3Resource) readWrittenResource(resource);
+ Assert.assertEquals(resource1.toString(), resource.toString());
+ resource1.readLock();
+ resource1.readUnlock();
+ }
+
+ @Test
+ public void testAzureResourcePersist() throws IOException {
+ Resource resource = new AzureResource("azure_resource");
+
Assert.assertTrue(resource.toString().contains("\"clazz\":\"AzureResource\""));
+
+ Resource readResource = readWrittenResource(resource);
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ @Test
+ public void testReadLegacyAzureResourceWithoutClazz() throws IOException {
+ String json = "{\"name\":\"legacy_azure_resource\",\"type\":\"AZURE\","
+ + "\"references\":{},\"id\":123,\"version\":0}";
+
+ Resource readResource = readResourceFromJson(json);
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("legacy_azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ @Test
+ public void testReadLegacyAzureResourceMgrWithoutClazz() throws
IOException {
+ String json =
"{\"nameToResource\":{\"legacy_azure_resource\":{\"name\":\"legacy_azure_resource\","
+ +
"\"type\":\"AZURE\",\"references\":{},\"id\":123,\"version\":0}}}";
+
+ ResourceMgr resourceMgr = readResourceMgrFromJson(json);
+ Resource readResource =
resourceMgr.getResource("legacy_azure_resource");
+ Assert.assertTrue(readResource instanceof AzureResource);
+ Assert.assertEquals("legacy_azure_resource", readResource.getName());
+ Assert.assertEquals(Resource.ResourceType.AZURE,
readResource.getType());
+ readResource.readLock();
+ readResource.readUnlock();
+ }
+
+ private Resource readWrittenResource(Resource resource) throws IOException
{
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ resource.write(dos);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ Resource readResource = Resource.read(dis);
+ dis.close();
+ return readResource;
+ }
+
+ private Resource readResourceFromJson(String json) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ Text.writeString(dos, json);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ Resource readResource = Resource.read(dis);
+ dis.close();
+ return readResource;
+ }
+
+ private ResourceMgr readResourceMgrFromJson(String json) throws
IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(byteArrayOutputStream);
+ Text.writeString(dos, json);
+ dos.flush();
+ dos.close();
+
+ DataInputStream dis = new DataInputStream(new
ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
+ ResourceMgr resourceMgr = ResourceMgr.read(dis);
+ dis.close();
+ return resourceMgr;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
