This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 19677eec78162598a5444b497a2b580c80cdb282 Author: Yongqiang YANG <[email protected]> AuthorDate: Thu Jul 4 16:35:53 2024 +0800 [fix](recyclebin) db can not be serialized with gson directly. (#37261) Do not serialize list in recyclebin too. It may takes memory beyond 2GB. --- .../org/apache/doris/common/FeMetaVersion.java | 4 +- .../apache/doris/catalog/CatalogRecycleBin.java | 114 ++++++++++++++++++--- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 919917e1ec5..4f2be870987 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -99,8 +99,10 @@ public final class FeMetaVersion { public static final int VERSION_138 = 138; + public static final int VERSION_139 = 139; + // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_138; + public static final int VERSION_CURRENT = VERSION_139; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 17cc5cd148b..4b24736f880 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -68,15 +68,15 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos // to avoid erase log ahead of drop log private static final long minEraseLatency = 10 * 60 * 1000; // 10 min - @SerializedName(value = "itd") private Map<Long, RecycleDatabaseInfo> idToDatabase; - @SerializedName(value = "itt") private Map<Long, RecycleTableInfo> idToTable; - @SerializedName(value = "itp") private Map<Long, RecyclePartitionInfo> idToPartition; - @SerializedName(value = "itr") private Map<Long, Long> idToRecycleTime; + // for compatible in the future + @SerializedName("u") + String unused; + public CatalogRecycleBin() { super("recycle bin", FeConstants.runningUnitTest ? 10L : DEFAULT_INTERVAL_SECONDS * 1000L); idToDatabase = Maps.newHashMap(); @@ -1287,17 +1287,75 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos // this class is not protected by any lock, will throw ConcurrentModificationException. @Override public synchronized void write(DataOutput out) throws IOException { + out.writeInt(idToDatabase.size()); + for (Map.Entry<Long, RecycleDatabaseInfo> entry : idToDatabase.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToTable.size()); + for (Map.Entry<Long, RecycleTableInfo> entry : idToTable.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToPartition.size()); + for (Map.Entry<Long, RecyclePartitionInfo> entry : idToPartition.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToRecycleTime.size()); + for (Map.Entry<Long, Long> entry : idToRecycleTime.entrySet()) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue()); + } Text.writeString(out, GsonUtils.GSON.toJson(this)); } + public void readFieldsWithGson(DataInput in) throws IOException { + int count = in.readInt(); + for (int i = 0; i < count; i++) { + long id = in.readLong(); + RecycleDatabaseInfo dbInfo = new RecycleDatabaseInfo(); + dbInfo.readFields(in); + idToDatabase.put(id, dbInfo); + } + + count = in.readInt(); + for (int i = 0; i < count; i++) { + long id = in.readLong(); + RecycleTableInfo tableInfo = new RecycleTableInfo(); + tableInfo = tableInfo.read(in); + idToTable.put(id, tableInfo); + } + + count = in.readInt(); + for (int i = 0; i < count; i++) { + long id = in.readLong(); + RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(); + partitionInfo = partitionInfo.read(in); + idToPartition.put(id, partitionInfo); + } + + count = in.readInt(); + for (int i = 0; i < count; i++) { + long id = in.readLong(); + long time = in.readLong(); + idToRecycleTime.put(id, time); + } + GsonUtils.GSON.fromJson(Text.readString(in), CatalogRecycleBin.class); + } + public static CatalogRecycleBin read(DataInput in) throws IOException { - if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_136) { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_136) { + CatalogRecycleBin bin = new CatalogRecycleBin(); + bin.readFields(in); + return bin; + } else if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_139) { return GsonUtils.GSON.fromJson(Text.readString(in), CatalogRecycleBin.class); + } else { + CatalogRecycleBin bin = new CatalogRecycleBin(); + bin.readFieldsWithGson(in); + return bin; } - - CatalogRecycleBin bin = new CatalogRecycleBin(); - bin.readFields(in); - return bin; } @Override @@ -1365,12 +1423,12 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos } public class RecycleDatabaseInfo { - @SerializedName("db") private Database db; - @SerializedName("tns") private Set<String> tableNames; - @SerializedName("tis") private Set<Long> tableIds; + // for compatibility in the future + @SerializedName("u") + private String unused; public RecycleDatabaseInfo() { tableNames = Sets.newHashSet(); @@ -1399,6 +1457,19 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos return this.tableIds = tableIds; } + public void write(DataOutput out) throws IOException { + db.write(out); + out.writeInt(tableNames.size()); + for (String tableName : tableNames) { + Text.writeString(out, tableName); + } + out.writeInt(tableIds.size()); + for (Long tableId : tableIds) { + out.writeLong(tableId); + } + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + public void readFields(DataInput in) throws IOException { db = Database.read(in); @@ -1414,6 +1485,9 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos tableIds.add(tableId); } } + if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_139) { + GsonUtils.GSON.fromJson(Text.readString(in), RecycleDatabaseInfo.class); + } } } @@ -1440,6 +1514,14 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos return table; } + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public RecycleTableInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), RecycleTableInfo.class); + } + @Deprecated public void readFields(DataInput in) throws IOException { dbId = in.readLong(); @@ -1522,6 +1604,14 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos return isMutable; } + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public RecyclePartitionInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), RecyclePartitionInfo.class); + } + public void readFields(DataInput in) throws IOException { dbId = in.readLong(); tableId = in.readLong(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
