This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ac7984ef3fc [improvement](meta) Serialize meta via gson part2 (#36480)
ac7984ef3fc is described below
commit ac7984ef3fcf61025919498df6eb33ae0b282201
Author: Yongqiang YANG <[email protected]>
AuthorDate: Fri Jun 21 17:32:21 2024 +0800
[improvement](meta) Serialize meta via gson part2 (#36480)
iszhangpch handles most of work.
dataroaring handles upgrading mtmv by introducing GsonUtils134.
Co-authored-by: iszhangpch
---
.../org/apache/doris/common/FeMetaVersion.java | 4 +-
.../doris/alter/BatchAlterJobPersistInfo.java | 26 +++++---
.../java/org/apache/doris/analysis/TableRef.java | 27 ++++----
.../java/org/apache/doris/backup/AbstractJob.java | 12 ++++
.../java/org/apache/doris/backup/BackupJob.java | 23 +++++--
.../org/apache/doris/backup/BackupJobInfo.java | 24 ++++---
.../java/org/apache/doris/backup/BackupMeta.java | 11 +++-
.../apache/doris/backup/RestoreFileMapping.java | 40 +++++------
.../java/org/apache/doris/backup/RestoreJob.java | 41 ++++++++++--
.../java/org/apache/doris/backup/SnapshotInfo.java | 38 ++++++-----
.../apache/doris/catalog/FunctionSearchDesc.java | 27 +++++---
.../main/java/org/apache/doris/catalog/MTMV.java | 13 +++-
.../org/apache/doris/persist/gson/GsonUtils.java | 77 +++++++++++++++++++---
.../gson/{GsonUtils.java => GsonUtils134.java} | 2 +-
.../persist/gson/RuntimeTypeAdapterFactory.java | 5 +-
.../org/apache/doris/backup/BackupJobTest.java | 35 ++++++++++
.../org/apache/doris/backup/RestoreJobTest.java | 30 +++++++++
17 files changed, 331 insertions(+), 104 deletions(-)
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
index 5f7b7991e00..a54d30d294e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
@@ -90,8 +90,10 @@ public final class FeMetaVersion {
public static final int VERSION_133 = 133;
// For mate gson
public static final int VERSION_134 = 134;
+ // For mate gson
+ public static final int VERSION_135 = 135;
// note: when increment meta version, should assign the latest version to
VERSION_CURRENT
- public static final int VERSION_CURRENT = VERSION_134;
+ public static final int VERSION_CURRENT = VERSION_135;
// all logs meta version should >= the minimum version, so that we could
remove many if clause, for example
// if (FE_METAVERSION < VERSION_94) ...
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/BatchAlterJobPersistInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/BatchAlterJobPersistInfo.java
index 3d6caefee50..b9313302c3c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/BatchAlterJobPersistInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/BatchAlterJobPersistInfo.java
@@ -17,7 +17,13 @@
package org.apache.doris.alter;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -32,6 +38,7 @@ import java.util.List;
*/
public class BatchAlterJobPersistInfo implements Writable {
+ @SerializedName("l")
private List<AlterJobV2> alterJobV2List;
public BatchAlterJobPersistInfo(List<AlterJobV2> alterJobV2List) {
@@ -40,19 +47,20 @@ public class BatchAlterJobPersistInfo implements Writable {
@Override
public void write(DataOutput out) throws IOException {
- out.writeInt(alterJobV2List.size());
- for (AlterJobV2 alterJobV2 : alterJobV2List) {
- alterJobV2.write(out);
- }
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
}
public static BatchAlterJobPersistInfo read(DataInput in) throws
IOException {
- int size = in.readInt();
- List<AlterJobV2> alterJobV2List = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- alterJobV2List.add(AlterJobV2.read(in));
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ int size = in.readInt();
+ List<AlterJobV2> alterJobV2List = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ alterJobV2List.add(AlterJobV2.read(in));
+ }
+ return new BatchAlterJobPersistInfo(alterJobV2List);
+ } else {
+ return GsonUtils.GSON.fromJson(Text.readString(in),
BatchAlterJobPersistInfo.class);
}
- return new BatchAlterJobPersistInfo(alterJobV2List);
}
public List<AlterJobV2> getAlterJobV2List() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
index 13821a510c6..3166d327d37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
@@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
@@ -33,6 +34,7 @@ import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hudi.HudiUtils;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.rewrite.ExprRewriter.ClauseType;
@@ -40,6 +42,7 @@ import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import com.google.gson.annotations.SerializedName;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -84,6 +87,7 @@ import java.util.regex.Matcher;
*/
public class TableRef implements ParseNode, Writable {
private static final Logger LOG = LogManager.getLogger(TableRef.class);
+ @SerializedName("n")
protected TableName name;
// Legal aliases of this table ref. Contains the explicit alias as its
sole element if
// there is one. Otherwise, contains the two implicit aliases. Implicit
aliases are set
@@ -91,6 +95,7 @@ public class TableRef implements ParseNode, Writable {
// analysis. By convention, for table refs with multiple implicit aliases,
aliases_[0]
// contains the fully-qualified implicit alias to ensure that aliases_[0]
always
// uniquely identifies this table ref regardless of whether it has an
explicit alias.
+ @SerializedName("a")
protected String[] aliases;
protected List<Long> sampleTabletIds;
// Indicates whether this table ref is given an explicit alias,
@@ -128,6 +133,7 @@ public class TableRef implements ParseNode, Writable {
protected List<TupleId> correlatedTupleIds = Lists.newArrayList();
// analysis output
protected TupleDescriptor desc;
+ @SerializedName("p")
private PartitionNames partitionNames = null;
private ArrayList<String> joinHints;
private ArrayList<String> sortHints;
@@ -973,23 +979,20 @@ public class TableRef implements ParseNode, Writable {
@Override
public void write(DataOutput out) throws IOException {
- name.write(out);
- if (partitionNames == null) {
- out.writeBoolean(false);
- } else {
- out.writeBoolean(true);
- partitionNames.write(out);
- }
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
- if (hasExplicitAlias()) {
- out.writeBoolean(true);
- Text.writeString(out, getExplicitAlias());
+ public static TableRef read(DataInput in) throws IOException {
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ TableRef ref = new TableRef();
+ ref.readFields(in);
+ return ref;
} else {
- out.writeBoolean(false);
+ return GsonUtils.GSON.fromJson(Text.readString(in),
TableRef.class);
}
}
- public void readFields(DataInput in) throws IOException {
+ private void readFields(DataInput in) throws IOException {
name = new TableName();
name.readFields(in);
if (in.readBoolean()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java
index 0df9155ab34..b29938eb9f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.io.Writable;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -43,12 +44,14 @@ public abstract class AbstractJob implements Writable {
BACKUP, RESTORE
}
+ @SerializedName("t")
protected JobType type;
// must be set right before job's running
protected Env env;
// repo will be set at first run()
protected Repository repo;
+ @SerializedName("rid")
protected long repoId;
/*
@@ -57,16 +60,23 @@ public abstract class AbstractJob implements Writable {
* And each time this method is called, the snapshot tasks will be sent
with (maybe) different
* version and version hash. So we have to use different job id to
identify the tasks in different batches.
*/
+ @SerializedName("jid")
protected long jobId = -1;
+ @SerializedName("l")
protected String label;
+ @SerializedName("dbid")
protected long dbId;
+ @SerializedName("dbn")
protected String dbName;
protected Status status = Status.OK;
+ @SerializedName("ct")
protected long createTime = -1;
+ @SerializedName("ft")
protected long finishedTime = -1;
+ @SerializedName("to")
protected long timeoutMs;
// task signature -> <finished num / total num>
@@ -75,6 +85,7 @@ public abstract class AbstractJob implements Writable {
protected boolean isTypeRead = false;
// save err msg of tasks
+ @SerializedName("msg")
protected Map<Long, String> taskErrMsg = Maps.newHashMap();
protected AbstractJob(JobType type) {
@@ -207,6 +218,7 @@ public abstract class AbstractJob implements Writable {
}
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
if (!isTypeRead) {
type = JobType.valueOf(Text.readString(in));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index fc846bf1820..89b5ccccb6e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -58,6 +58,7 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -92,18 +93,24 @@ public class BackupJob extends AbstractJob {
}
// all objects which need backup
+ @SerializedName("ref")
private List<TableRef> tableRefs = Lists.newArrayList();
+ @SerializedName("st")
private BackupJobState state;
+ @SerializedName("sft")
private long snapshotFinishedTime = -1;
+ @SerializedName("suft")
private long snapshotUploadFinishedTime = -1;
// save task id map to the backend it be executed
private Map<Long, Long> unfinishedTaskIds = Maps.newConcurrentMap();
// tablet id -> snapshot info
+ @SerializedName("si")
private Map<Long, SnapshotInfo> snapshotInfos = Maps.newConcurrentMap();
// save all related table[partition] info
+ @SerializedName("meta")
private BackupMeta backupMeta;
// job info file content
private BackupJobInfo jobInfo;
@@ -112,9 +119,12 @@ public class BackupJob extends AbstractJob {
// after job is done, this dir should be deleted
private Path localJobDirPath = null;
// save the local file path of meta info and job info file
+ @SerializedName("mifp")
private String localMetaInfoFilePath = null;
+ @SerializedName("jifp")
private String localJobInfoFilePath = null;
// backup properties && table commit seq with table id
+ @SerializedName("prop")
private Map<String, String> properties = Maps.newHashMap();
private byte[] metaInfoBytes = null;
@@ -1033,10 +1043,11 @@ public class BackupJob extends AbstractJob {
// table refs
int size = in.readInt();
+ LOG.info("read {} tablerefs ", size);
+
tableRefs = Lists.newArrayList();
for (int i = 0; i < size; i++) {
- TableRef tblRef = new TableRef();
- tblRef.readFields(in);
+ TableRef tblRef = TableRef.read(in);
tableRefs.add(tblRef);
}
@@ -1048,14 +1059,16 @@ public class BackupJob extends AbstractJob {
// snapshot info
size = in.readInt();
+ LOG.info("read {} snapshotinfo ", size);
+
for (int i = 0; i < size; i++) {
- SnapshotInfo snapshotInfo = new SnapshotInfo();
- snapshotInfo.readFields(in);
+ SnapshotInfo snapshotInfo = SnapshotInfo.read(in);
snapshotInfos.put(snapshotInfo.getTabletId(), snapshotInfo);
}
// backup meta
if (in.readBoolean()) {
+ LOG.info("read backup meta");
backupMeta = BackupMeta.read(in);
}
@@ -1071,6 +1084,8 @@ public class BackupJob extends AbstractJob {
}
// read properties
size = in.readInt();
+ LOG.info("read {} property ", size);
+
for (int i = 0; i < size; i++) {
String key = Text.readString(in);
String value = Text.readString(in);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
index aa127961e3d..554a21c4408 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.BackupStmt.BackupContent;
import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.backup.RestoreFileMapping.IdChain;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.OdbcCatalogResource;
@@ -34,9 +35,11 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Version;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TNetworkAddress;
@@ -69,7 +72,7 @@ import java.util.Set;
* It contains all content of a job info file.
* It also be used to save the info of a restore job, such as alias of table
and meta info file path
*/
-public class BackupJobInfo implements Writable {
+public class BackupJobInfo implements Writable, GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(BackupJobInfo.class);
@SerializedName("name")
@@ -133,6 +136,7 @@ public class BackupJobInfo implements Writable {
// This map is used to save the table alias mapping info when processing a
restore job.
// origin -> alias
+ @SerializedName("tblalias")
public Map<String, String> tblAlias = Maps.newHashMap();
public long getBackupTime() {
@@ -756,11 +760,9 @@ public class BackupJobInfo implements Writable {
* }
*/
BackupJobInfo jobInfo = GsonUtils.GSON.fromJson(json,
BackupJobInfo.class);
- jobInfo.initBackupJobInfoAfterDeserialize();
return jobInfo;
}
-
public void writeToFile(File jobInfoFile) throws FileNotFoundException {
PrintWriter printWriter = new PrintWriter(jobInfoFile);
try {
@@ -807,17 +809,21 @@ public class BackupJobInfo implements Writable {
}
public static BackupJobInfo read(DataInput in) throws IOException {
- return BackupJobInfo.readFields(in);
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ return BackupJobInfo.readFields(in);
+ }
+ String json = Text.readString(in);
+ return genFromJson(json);
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, toJson(false));
- out.writeInt(tblAlias.size());
- for (Map.Entry<String, String> entry : tblAlias.entrySet()) {
- Text.writeString(out, entry.getKey());
- Text.writeString(out, entry.getValue());
- }
+ }
+
+ @Override
+ public void gsonPostProcess() throws IOException {
+ initBackupJobInfoAfterDeserialize();
}
public static BackupJobInfo readFields(DataInput in) throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
index e22bb7f33ce..e9a1866a59b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.io.Writable;
import org.apache.doris.meta.MetaContext;
+import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.collect.Maps;
@@ -37,13 +38,12 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
-public class BackupMeta implements Writable {
+public class BackupMeta implements Writable, GsonPostProcessable {
// tbl name -> tbl
@SerializedName(value = "tblNameMap")
private Map<String, Table> tblNameMap = Maps.newHashMap();
// tbl id -> tbl
- @SerializedName(value = "tblIdMap")
private Map<Long, Table> tblIdMap = Maps.newHashMap();
// resource name -> resource
@SerializedName(value = "resourceNameMap")
@@ -142,6 +142,13 @@ public class BackupMeta implements Writable {
}
}
+ @Override
+ public void gsonPostProcess() throws IOException {
+ for (Table table : tblNameMap.values()) {
+ tblIdMap.put(table.getId(), table);
+ }
+ }
+
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
index 07ddf6844dc..4bb791d84d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
@@ -17,11 +17,16 @@
package org.apache.doris.backup;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -30,8 +35,9 @@ import java.util.Map;
public class RestoreFileMapping implements Writable {
- public static class IdChain implements Writable {
+ public static class IdChain {
// tblId, partId, idxId, tabletId, replicaId
+ @SerializedName("c")
private Long[] chain;
private IdChain() {
@@ -98,14 +104,6 @@ public class RestoreFileMapping implements Writable {
return code;
}
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(chain.length);
- for (Long id : chain) {
- out.writeLong(id);
- }
- }
-
public void readFields(DataInput in) throws IOException {
int size = in.readInt();
chain = new Long[size];
@@ -122,8 +120,10 @@ public class RestoreFileMapping implements Writable {
}
// catalog ids -> repository ids
+ @SerializedName("m")
private Map<IdChain, IdChain> mapping = Maps.newHashMap();
// tablet id -> is overwrite
+ @SerializedName("o")
private Map<Long, Boolean> overwriteMap = Maps.newHashMap();
public RestoreFileMapping() {
@@ -151,9 +151,13 @@ public class RestoreFileMapping implements Writable {
}
public static RestoreFileMapping read(DataInput in) throws IOException {
- RestoreFileMapping mapping = new RestoreFileMapping();
- mapping.readFields(in);
- return mapping;
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ RestoreFileMapping mapping = new RestoreFileMapping();
+ mapping.readFields(in);
+ return mapping;
+ } else {
+ return GsonUtils.GSON.fromJson(Text.readString(in),
RestoreFileMapping.class);
+ }
}
public void clear() {
@@ -163,17 +167,7 @@ public class RestoreFileMapping implements Writable {
@Override
public void write(DataOutput out) throws IOException {
- out.writeInt(mapping.size());
- for (Map.Entry<IdChain, IdChain> entry : mapping.entrySet()) {
- entry.getKey().write(out);
- entry.getValue().write(out);
- }
-
- out.writeInt(overwriteMap.size());
- for (Map.Entry<Long, Boolean> entry : overwriteMap.entrySet()) {
- out.writeLong(entry.getKey());
- out.writeBoolean(entry.getValue());
- }
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
}
public void readFields(DataInput in) throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 99f5f028e6e..903e2ef95ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -66,6 +66,8 @@ import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.S3ClientBEProperties;
+import org.apache.doris.persist.gson.GsonPostProcessable;
+// import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.task.AgentBatchTask;
import org.apache.doris.task.AgentTask;
@@ -93,6 +95,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Table.Cell;
+import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -106,7 +109,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-public class RestoreJob extends AbstractJob {
+public class RestoreJob extends AbstractJob implements GsonPostProcessable {
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE =
"reserve_dynamic_partition_enable";
private static final String PROP_IS_BEING_SYNCED =
PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED;
@@ -129,21 +132,31 @@ public class RestoreJob extends AbstractJob {
}
// CHECKSTYLE ON
+ @SerializedName("bts")
private String backupTimestamp;
+ @SerializedName("j")
private BackupJobInfo jobInfo;
+ @SerializedName("al")
private boolean allowLoad;
+ @SerializedName("st")
private RestoreJobState state;
+ @SerializedName("meta")
private BackupMeta backupMeta;
+ @SerializedName("fm")
private RestoreFileMapping fileMapping = new RestoreFileMapping();
+ @SerializedName("mpt")
private long metaPreparedTime = -1;
+ @SerializedName("sft")
private long snapshotFinishedTime = -1;
+ @SerializedName("dft")
private long downloadFinishedTime = -1;
+ @SerializedName("ra")
private ReplicaAllocation replicaAlloc;
private boolean reserveReplica = false;
@@ -151,14 +164,19 @@ public class RestoreJob extends AbstractJob {
// this 2 members is to save all newly restored objs
// tbl name -> part
+ @SerializedName("rp")
private List<Pair<String, Partition>> restoredPartitions =
Lists.newArrayList();
+ @SerializedName("rt")
private List<Table> restoredTbls = Lists.newArrayList();
+ @SerializedName("rr")
private List<Resource> restoredResources = Lists.newArrayList();
// save all restored partitions' version info which are already exist in
catalog
// table id -> partition id -> (version, version hash)
+ @SerializedName("rvi")
private com.google.common.collect.Table<Long, Long, Long>
restoredVersionInfo = HashBasedTable.create();
// tablet id->(be id -> snapshot info)
+ @SerializedName("si")
private com.google.common.collect.Table<Long, Long, SnapshotInfo>
snapshotInfos = HashBasedTable.create();
private Map<Long, Long> unfinishedSignatureToId = Maps.newConcurrentMap();
@@ -175,6 +193,7 @@ public class RestoreJob extends AbstractJob {
private boolean isBeingSynced = false;
// restore properties
+ @SerializedName("prop")
private Map<String, String> properties = Maps.newHashMap();
public RestoreJob() {
@@ -2078,12 +2097,6 @@ public class RestoreJob extends AbstractJob {
}
}
- public static RestoreJob read(DataInput in) throws IOException {
- RestoreJob job = new RestoreJob();
- job.readFields(in);
- return job;
- }
-
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
@@ -2157,6 +2170,13 @@ public class RestoreJob extends AbstractJob {
}
}
+ public static RestoreJob read(DataInput in) throws IOException {
+ RestoreJob job = new RestoreJob();
+ job.readFields(in);
+ return job;
+ }
+
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -2238,6 +2258,13 @@ public class RestoreJob extends AbstractJob {
isBeingSynced =
Boolean.parseBoolean(properties.get(PROP_IS_BEING_SYNCED));
}
+ @Override
+ public void gsonPostProcess() throws IOException {
+ reserveReplica =
Boolean.parseBoolean(properties.get(PROP_RESERVE_REPLICA));
+ reserveDynamicPartitionEnable =
Boolean.parseBoolean(properties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
+ isBeingSynced =
Boolean.parseBoolean(properties.get(PROP_IS_BEING_SYNCED));
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/SnapshotInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/SnapshotInfo.java
index 60847e822dc..1d70babb98f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/SnapshotInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/SnapshotInfo.java
@@ -17,11 +17,15 @@
package org.apache.doris.backup;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -29,19 +33,28 @@ import java.io.IOException;
import java.util.List;
public class SnapshotInfo implements Writable {
+ @SerializedName("db")
private long dbId;
+ @SerializedName("tbl")
private long tblId;
+ @SerializedName("p")
private long partitionId;
+ @SerializedName("ind")
private long indexId;
+ @SerializedName("tab")
private long tabletId;
+ @SerializedName("be")
private long beId;
+ @SerializedName("sh")
private int schemaHash;
// eg: /path/to/your/be/data/snapshot/20180410102311.0.86400/
+ @SerializedName("path")
private String path;
// eg:
// 10006_0_1_0_0.dat
// 10006_2_2_0_0.idx
// 10006.hdr
+ @SerializedName("f")
private List<String> files = Lists.newArrayList();
public SnapshotInfo() {
@@ -107,26 +120,19 @@ public class SnapshotInfo implements Writable {
}
public static SnapshotInfo read(DataInput in) throws IOException {
- SnapshotInfo info = new SnapshotInfo();
- info.readFields(in);
- return info;
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ SnapshotInfo info = new SnapshotInfo();
+ info.readFields(in);
+ return info;
+ } else {
+ String json = Text.readString(in);
+ return GsonUtils.GSON.fromJson(json, SnapshotInfo.class);
+ }
}
@Override
public void write(DataOutput out) throws IOException {
- out.writeLong(dbId);
- out.writeLong(tblId);
- out.writeLong(partitionId);
- out.writeLong(indexId);
- out.writeLong(tabletId);
- out.writeLong(beId);
- out.writeInt(schemaHash);
- Text.writeString(out, path);
-
- out.writeInt(files.size());
- for (String file : files) {
- Text.writeString(out, file);
- }
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
}
public void readFields(DataInput in) throws IOException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSearchDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSearchDesc.java
index 3eed1ab4996..6e87e83b649 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSearchDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSearchDesc.java
@@ -18,7 +18,12 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.FunctionName;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -26,8 +31,11 @@ import java.io.IOException;
// Used to search a function
public class FunctionSearchDesc implements Writable {
+ @SerializedName("n")
private FunctionName name;
+ @SerializedName("t")
private Type[] argTypes;
+ @SerializedName("isV")
private boolean isVariadic;
private FunctionSearchDesc() {}
@@ -93,15 +101,10 @@ public class FunctionSearchDesc implements Writable {
@Override
public void write(DataOutput out) throws IOException {
- name.write(out);
- // write args
- out.writeShort(argTypes.length);
- for (Type type : argTypes) {
- ColumnType.write(out, type);
- }
- out.writeBoolean(isVariadic);
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
name = FunctionName.read(in);
// read args
@@ -114,8 +117,12 @@ public class FunctionSearchDesc implements Writable {
}
public static FunctionSearchDesc read(DataInput input) throws IOException {
- FunctionSearchDesc function = new FunctionSearchDesc();
- function.readFields(input);
- return function;
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ FunctionSearchDesc function = new FunctionSearchDesc();
+ function.readFields(input);
+ return function;
+ } else {
+ return GsonUtils.GSON.fromJson(Text.readString(input),
FunctionSearchDesc.class);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 1b4a8e7063e..f238cf2f1ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.catalog.OlapTableFactory.MTMVParams;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.PropertyAnalyzer;
@@ -41,10 +42,14 @@ import org.apache.doris.mtmv.MTMVRefreshSnapshot;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.persist.gson.GsonUtils134;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+//import com.google.gson.JsonElement;
+//import com.google.gson.JsonObject;
+//import com.google.gson.JsonParser;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
@@ -453,7 +458,13 @@ public class MTMV extends OlapTable {
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- MTMV materializedView = GsonUtils.GSON.fromJson(Text.readString(in),
this.getClass());
+ MTMV materializedView = null;
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_135) {
+ materializedView = GsonUtils134.GSON.fromJson(Text.readString(in),
this.getClass());
+ } else {
+ materializedView = GsonUtils.GSON.fromJson(Text.readString(in),
this.getClass());
+ }
+
refreshInfo = materializedView.refreshInfo;
querySql = materializedView.querySql;
status = materializedView.status;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index 482118bb4ee..79775a2ef24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -58,19 +58,35 @@ import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.StructLiteral;
import org.apache.doris.analysis.TimestampArithmeticExpr;
import org.apache.doris.analysis.VirtualSlotRef;
+import org.apache.doris.backup.BackupJob;
+import org.apache.doris.backup.RestoreJob;
import org.apache.doris.catalog.AggStateType;
+import org.apache.doris.catalog.AnyElementType;
+import org.apache.doris.catalog.AnyStructType;
+import org.apache.doris.catalog.AnyType;
import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.BrokerTable;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsResource;
+import org.apache.doris.catalog.EsTable;
+import org.apache.doris.catalog.FunctionGenTable;
import org.apache.doris.catalog.HMSResource;
import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.HdfsResource;
+import org.apache.doris.catalog.HiveTable;
+import org.apache.doris.catalog.InlineView;
import org.apache.doris.catalog.JdbcResource;
+import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.catalog.ListPartitionInfo;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.MultiRowType;
+import org.apache.doris.catalog.MysqlDBTable;
+import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcCatalogResource;
+import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
@@ -81,11 +97,15 @@ import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.S3Resource;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.catalog.SparkResource;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.catalog.TemplateType;
+import org.apache.doris.catalog.VariantType;
+import org.apache.doris.catalog.View;
import org.apache.doris.catalog.constraint.Constraint;
import org.apache.doris.catalog.constraint.ForeignKeyConstraint;
import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
@@ -139,7 +159,13 @@ import org.apache.doris.datasource.test.TestExternalTable;
import
org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalog;
import
org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase;
import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalTable;
-import org.apache.doris.job.base.AbstractJob;
+import org.apache.doris.fs.remote.BrokerFileSystem;
+import org.apache.doris.fs.remote.ObjFileSystem;
+import org.apache.doris.fs.remote.RemoteFileSystem;
+import org.apache.doris.fs.remote.S3FileSystem;
+import org.apache.doris.fs.remote.dfs.DFSFileSystem;
+import org.apache.doris.fs.remote.dfs.JFSFileSystem;
+import org.apache.doris.fs.remote.dfs.OFSFileSystem;
import org.apache.doris.job.extensions.insert.InsertJob;
import org.apache.doris.job.extensions.mtmv.MTMVJob;
import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
@@ -236,7 +262,13 @@ public class GsonUtils {
.registerSubtype(ArrayType.class, ArrayType.class.getSimpleName())
.registerSubtype(MapType.class, MapType.class.getSimpleName())
.registerSubtype(StructType.class,
StructType.class.getSimpleName())
- .registerSubtype(AggStateType.class,
AggStateType.class.getSimpleName());
+ .registerSubtype(AggStateType.class,
AggStateType.class.getSimpleName())
+ .registerSubtype(AnyElementType.class,
AnyElementType.class.getSimpleName())
+ .registerSubtype(AnyStructType.class,
AnyStructType.class.getSimpleName())
+ .registerSubtype(AnyType.class, AnyType.class.getSimpleName())
+ .registerSubtype(MultiRowType.class,
MultiRowType.class.getSimpleName())
+ .registerSubtype(TemplateType.class,
TemplateType.class.getSimpleName())
+ .registerSubtype(VariantType.class,
VariantType.class.getSimpleName());
// runtime adapter for class "Expr"
private static final
RuntimeTypeAdapterFactory<org.apache.doris.analysis.Expr> exprAdapterFactory
@@ -373,10 +405,11 @@ public class GsonUtils {
RuntimeTypeAdapterFactory.of(
AbstractDataSourceProperties.class, "clazz")
.registerSubtype(KafkaDataSourceProperties.class,
KafkaDataSourceProperties.class.getSimpleName());
- private static RuntimeTypeAdapterFactory<AbstractJob>
jobExecutorRuntimeTypeAdapterFactory =
- RuntimeTypeAdapterFactory.of(AbstractJob.class, "clazz")
- .registerSubtype(InsertJob.class,
InsertJob.class.getSimpleName())
- .registerSubtype(MTMVJob.class,
MTMVJob.class.getSimpleName());
+ private static
RuntimeTypeAdapterFactory<org.apache.doris.job.base.AbstractJob>
+ jobExecutorRuntimeTypeAdapterFactory
+ =
RuntimeTypeAdapterFactory.of(org.apache.doris.job.base.AbstractJob.class,
"clazz")
+ .registerSubtype(InsertJob.class,
InsertJob.class.getSimpleName())
+ .registerSubtype(MTMVJob.class,
MTMVJob.class.getSimpleName());
private static RuntimeTypeAdapterFactory<MTMVSnapshotIf>
mtmvSnapshotTypeAdapterFactory =
RuntimeTypeAdapterFactory.of(MTMVSnapshotIf.class, "clazz")
@@ -412,7 +445,19 @@ public class GsonUtils {
.registerSubtype(ExternalInfoSchemaTable.class,
ExternalInfoSchemaTable.class.getSimpleName())
.registerSubtype(ExternalMysqlTable.class,
ExternalMysqlTable.class.getSimpleName())
.registerSubtype(TrinoConnectorExternalTable.class,
TrinoConnectorExternalTable.class.getSimpleName())
- .registerSubtype(TestExternalTable.class,
TestExternalTable.class.getSimpleName());
+ .registerSubtype(TestExternalTable.class,
TestExternalTable.class.getSimpleName())
+ .registerSubtype(BrokerTable.class,
BrokerTable.class.getSimpleName())
+ .registerSubtype(EsTable.class, EsTable.class.getSimpleName())
+ .registerSubtype(FunctionGenTable.class,
FunctionGenTable.class.getSimpleName())
+ .registerSubtype(HiveTable.class, HiveTable.class.getSimpleName())
+ .registerSubtype(InlineView.class,
InlineView.class.getSimpleName())
+ .registerSubtype(JdbcTable.class, JdbcTable.class.getSimpleName())
+ .registerSubtype(MTMV.class, MTMV.class.getSimpleName())
+ .registerSubtype(MysqlDBTable.class,
MysqlDBTable.class.getSimpleName())
+ .registerSubtype(MysqlTable.class,
MysqlTable.class.getSimpleName())
+ .registerSubtype(OdbcTable.class, OdbcTable.class.getSimpleName())
+ .registerSubtype(SchemaTable.class,
SchemaTable.class.getSimpleName())
+ .registerSubtype(View.class, View.class.getSimpleName());
// runtime adapter for class "PartitionInfo"
private static RuntimeTypeAdapterFactory<PartitionInfo>
partitionInfoTypeAdapterFactory
@@ -471,6 +516,21 @@ public class GsonUtils {
.registerDefaultSubtype(RoutineLoadProgress.class)
.registerSubtype(KafkaProgress.class,
KafkaProgress.class.getSimpleName());
+ private static RuntimeTypeAdapterFactory<RemoteFileSystem>
remoteFileSystemTypeAdapterFactory
+ = RuntimeTypeAdapterFactory.of(RemoteFileSystem.class, "clazz")
+ .registerSubtype(BrokerFileSystem.class,
BrokerFileSystem.class.getSimpleName())
+ .registerSubtype(DFSFileSystem.class,
DFSFileSystem.class.getSimpleName())
+ .registerSubtype(JFSFileSystem.class,
JFSFileSystem.class.getSimpleName())
+ .registerSubtype(OFSFileSystem.class,
OFSFileSystem.class.getSimpleName())
+ .registerSubtype(ObjFileSystem.class,
ObjFileSystem.class.getSimpleName())
+ .registerSubtype(S3FileSystem.class,
S3FileSystem.class.getSimpleName());
+
+ private static
RuntimeTypeAdapterFactory<org.apache.doris.backup.AbstractJob>
+ jobBackupTypeAdapterFactory
+ =
RuntimeTypeAdapterFactory.of(org.apache.doris.backup.AbstractJob.class, "clazz")
+ .registerSubtype(BackupJob.class,
BackupJob.class.getSimpleName())
+ .registerSubtype(RestoreJob.class,
RestoreJob.class.getSimpleName());
+
// the builder of GSON instance.
// Add any other adapters if necessary.
private static final GsonBuilder GSON_BUILDER = new
GsonBuilder().addSerializationExclusionStrategy(
@@ -501,6 +561,8 @@ public class GsonUtils {
.registerTypeAdapterFactory(constraintTypeAdapterFactory)
.registerTypeAdapterFactory(txnCommitAttachmentTypeAdapterFactory)
.registerTypeAdapterFactory(routineLoadTypeAdapterFactory)
+ .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactory)
+ .registerTypeAdapterFactory(jobBackupTypeAdapterFactory)
.registerTypeAdapter(ImmutableMap.class, new
ImmutableMapDeserializer())
.registerTypeAdapter(AtomicBoolean.class, new
AtomicBooleanAdapter())
.registerTypeAdapter(PartitionKey.class, new
PartitionKey.PartitionKeySerializer())
@@ -656,7 +718,6 @@ public class GsonUtils {
private static final String EXPR_PROP = "expr";
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
-
final Class<T> rawType = (Class<T>) type.getRawType();
final TypeAdapter<T> delegate = gson.getDelegateAdapter(this,
type);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils134.java
similarity index 99%
copy from fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
copy to fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils134.java
index 482118bb4ee..1359bb170df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils134.java
@@ -225,7 +225,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
* You need implement a class to implements JsonSerializer and
JsonDeserializer, and register it to GSON_BUILDER.
* See the following "GuavaTableAdapter" and "GuavaMultimapAdapter" for
example.
*/
-public class GsonUtils {
+public class GsonUtils134 {
// runtime adapter for class "Type"
private static RuntimeTypeAdapterFactory<org.apache.doris.catalog.Type>
columnTypeAdapterFactory
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java
index 15270bc85bb..ee780f0de1b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java
@@ -32,6 +32,7 @@ import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
+import java.lang.reflect.Modifier;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -282,7 +283,9 @@ public final class RuntimeTypeAdapterFactory<T> implements
TypeAdapterFactory {
}
public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
- if (type.getRawType() != baseType &&
!subtypeToLabel.containsKey(type.getRawType())) {
+ if (baseType != type.getRawType() &&
!subtypeToLabel.containsKey(type.getRawType())
+ && !(Modifier.isAbstract(type.getRawType().getModifiers())
+ && baseType.isAssignableFrom(type.getRawType()))) {
return null;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
index 4e0eecda1fa..68f02e8eee8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
@@ -26,6 +26,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.FsBroker;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
@@ -58,6 +59,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitOption;
@@ -353,4 +356,36 @@ public class BackupJobTest {
Assert.assertEquals(Status.ErrCode.NOT_FOUND,
job.getStatus().getErrCode());
Assert.assertEquals(BackupJobState.CANCELLED, job.getState());
}
+
+ @Test
+ public void testSerialization() throws IOException, AnalysisException {
+ // 1. Write objects to file
+ final Path path = Files.createTempFile("backupJob", "tmp");
+ DataOutputStream out = new
DataOutputStream(Files.newOutputStream(path));
+
+ List<TableRef> tableRefs = Lists.newArrayList();
+ tableRefs.add(
+ new TableRef(new
TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME,
UnitTestUtil.TABLE_NAME),
+ null));
+ job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs,
13600 * 1000, BackupStmt.BackupContent.ALL,
+ env, repo.getId());
+
+ job.write(out);
+ out.flush();
+ out.close();
+
+ // 2. Read objects from file
+ DataInputStream in = new DataInputStream(Files.newInputStream(path));
+
+ BackupJob job2 = BackupJob.read(in);
+
+ Assert.assertEquals(job.getJobId(), job2.getJobId());
+ Assert.assertEquals(job.getDbId(), job2.getDbId());
+ Assert.assertEquals(job.getCreateTime(), job2.getCreateTime());
+ Assert.assertEquals(job.getType(), job2.getType());
+
+ // 3. delete files
+ in.close();
+ Files.delete(path);
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index ff60a6e8b90..71cad0438c9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -51,9 +51,15 @@ import mockit.Injectable;
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -276,4 +282,28 @@ public class RestoreJobTest {
System.out.println("tbl signature: " +
tbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames));
}
+ @Test
+ public void testSerialization() throws IOException, AnalysisException {
+ // 1. Write objects to file
+ final Path path = Files.createTempFile("restoreJob", "tmp");
+ DataOutputStream out = new
DataOutputStream(Files.newOutputStream(path));
+
+ job.write(out);
+ out.flush();
+ out.close();
+
+ // 2. Read objects from file
+ DataInputStream in = new DataInputStream(Files.newInputStream(path));
+
+ RestoreJob job2 = RestoreJob.read(in);
+
+ Assert.assertEquals(job.getJobId(), job2.getJobId());
+ Assert.assertEquals(job.getDbId(), job2.getDbId());
+ Assert.assertEquals(job.getCreateTime(), job2.getCreateTime());
+ Assert.assertEquals(job.getType(), job2.getType());
+
+ // 3. delete files
+ in.close();
+ Files.delete(path);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
