This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ac20c02291ca3c1fde3ff6f9963a752997102c2f Author: Yongqiang YANG <[email protected]> AuthorDate: Wed Jul 3 21:38:04 2024 +0800 [fix](meta) do not serialize nametotable map (#37222) Gson can not handle size beyonds 2GB. --- .../org/apache/doris/common/FeMetaVersion.java | 4 ++- .../java/org/apache/doris/catalog/Database.java | 31 ++++++++++++++++++++-- .../java/org/apache/doris/catalog/OlapTable.java | 24 ++++++++--------- .../java/org/apache/doris/catalog/Replica.java | 22 +++++++-------- .../main/java/org/apache/doris/catalog/Tablet.java | 12 ++++----- 5 files changed, 61 insertions(+), 32 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 7aa98625639..919917e1ec5 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -97,8 +97,10 @@ public final class FeMetaVersion { // For mate gson public static final int VERSION_137 = 137; + public static final int VERSION_138 = 138; + // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_137; + public static final int VERSION_CURRENT = VERSION_138; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index a0f5527ef6d..5637e27e0d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -91,7 +91,6 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>, // table family group map private final Map<Long, Table> idToTable; - @SerializedName(value = "nameToTable") private ConcurrentMap<String, Table> nameToTable; // table name lower cast -> table name private final Map<String, String> lowerCaseToTableName; @@ -595,8 +594,35 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>, Database db = new Database(); db.readFields(in); return db; - } else { + } else if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_138) { return GsonUtils.GSON.fromJson(Text.readString(in), Database.class); + } else { + Database db = GsonUtils.GSON.fromJson(Text.readString(in), Database.class); + db.readTables(in); + return db; + } + } + + private void writeTables(DataOutput out) throws IOException { + out.writeInt(nameToTable.size()); + for (Table table : nameToTable.values()) { + Text.writeString(out, GsonUtils.GSON.toJson(table)); + } + } + + private void readTables(DataInput in) throws IOException { + nameToTable = Maps.newConcurrentMap(); + int numTables = in.readInt(); + for (int i = 0; i < numTables; ++i) { + Table table = Table.read(in); + table.setQualifiedDbName(fullQualifiedName); + if (table instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id); + } + String tableName = table.getName(); + nameToTable.put(tableName, table); + idToTable.put(table.getId(), table); + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } @@ -650,6 +676,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>, public void write(DataOutput out) throws IOException { discardHudiTable(); Text.writeString(out, GsonUtils.GSON.toJson(this)); + writeTables(out); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d5b1c258c5d..7d3b9baf341 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -135,44 +135,44 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc WAITING_STABLE } - @SerializedName(value = "state") + @SerializedName(value = "tst", alternate = {"state"}) private volatile OlapTableState state; // index id -> index meta - @SerializedName("indexIdToMeta") + @SerializedName(value = "itm", alternate = {"indexIdToMeta"}) private Map<Long, MaterializedIndexMeta> indexIdToMeta = Maps.newHashMap(); // index name -> index id - @SerializedName("indexNameToId") + @SerializedName(value = "inti", alternate = {"indexNameToId"}) private Map<String, Long> indexNameToId = Maps.newHashMap(); - @SerializedName("keysType") + @SerializedName(value = "kt", alternate = {"keysType"}) private KeysType keysType; @Setter - @SerializedName("partitionInfo") + @SerializedName(value = "pi", alternate = {"partitionInfo"}) private PartitionInfo partitionInfo; - @SerializedName("idToPartition") + @SerializedName(value = "itp", alternate = {"idToPartition"}) @Getter private ConcurrentHashMap<Long, Partition> idToPartition = new ConcurrentHashMap<>(); // handled in postgsonprocess @Getter private Map<String, Partition> nameToPartition = Maps.newTreeMap(); - @SerializedName(value = "distributionInfo") + @SerializedName(value = "di", alternate = {"distributionInfo"}) private DistributionInfo defaultDistributionInfo; // all info about temporary partitions are save in "tempPartitions" @Getter - @SerializedName(value = "tempPartitions") + @SerializedName(value = "tps", alternate = {"tempPartitions"}) private TempPartitions tempPartitions = new TempPartitions(); // bloom filter columns - @SerializedName(value = "bfColumns") + @SerializedName(value = "bfc", alternate = {"bfColumns"}) private Set<String> bfColumns; @SerializedName(value = "bfFpp") private double bfFpp; - @SerializedName(value = "colocateGroup") + @SerializedName(value = "cgs", alternate = "colocateGroup") private String colocateGroup; private boolean hasSequenceCol; @@ -187,10 +187,10 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc // So we add this 'baseIndexId' to explicitly specify the base index id, // which should be different with table id. // The init value is -1, which means there is not partition and index at all. - @SerializedName(value = "baseIndexId") + @SerializedName(value = "bid", alternate = {"baseIndexId"}) private long baseIndexId = -1; - @SerializedName(value = "tableProperty") + @SerializedName(value = "tp", alternate = {"tableProperty"}) private TableProperty tableProperty; @SerializedName(value = "aIncg") diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java index 22813ef5118..0fcbef00743 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java @@ -88,37 +88,37 @@ public class Replica { @SerializedName(value = "id") private long id; - @SerializedName(value = "backendId") + @SerializedName(value = "bid", alternate = {"backendId"}) private long backendId; // the version could be queried - @SerializedName(value = "version") + @SerializedName(value = "v", alternate = {"version"}) private volatile long version; @Deprecated - @SerializedName(value = "versionHash") + @SerializedName(value = "vh", alternate = {"versionHash"}) private long versionHash = 0L; private int schemaHash = -1; - @SerializedName(value = "dataSize") + @SerializedName(value = "ds", alternate = {"dataSize"}) private volatile long dataSize = 0; - @SerializedName(value = "remoteDataSize") + @SerializedName(value = "rds", alternate = {"remoteDataSize"}) private volatile long remoteDataSize = 0; - @SerializedName(value = "rowCount") + @SerializedName(value = "rc", alternate = {"rowCount"}) private volatile long rowCount = 0; - @SerializedName(value = "state") + @SerializedName(value = "st", alternate = {"state"}) private volatile ReplicaState state; // the last load failed version - @SerializedName(value = "lastFailedVersion") + @SerializedName(value = "lfv", alternate = {"lastFailedVersion"}) private long lastFailedVersion = -1L; @Deprecated - @SerializedName(value = "lastFailedVersionHash") + @SerializedName(value = "lfvh", alternate = {"lastFailedVersionHash"}) private long lastFailedVersionHash = 0L; // not serialized, not very important private long lastFailedTimestamp = 0; // the last load successful version - @SerializedName(value = "lastSuccessVersion") + @SerializedName(value = "lsv", alternate = {"lastSuccessVersion"}) private long lastSuccessVersion = -1L; @Deprecated - @SerializedName(value = "lastSuccessVersionHash") + @SerializedName(value = "lsvh", alternate = {"lastSuccessVersionHash"}) private long lastSuccessVersionHash = 0L; private volatile long totalVersionCount = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index 899eb94b8d0..3266d0eeb66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -83,20 +83,20 @@ public class Tablet extends MetaObject { @SerializedName(value = "id") protected long id; - @SerializedName(value = "replicas") + @SerializedName(value = "rs", alternate = {"replicas"}) protected List<Replica> replicas; - @SerializedName(value = "checkedVersion") + @SerializedName(value = "cv", alternate = {"checkedVersion"}) private long checkedVersion; @Deprecated - @SerializedName(value = "checkedVersionHash") + @SerializedName(value = "cvs", alternate = {"checkedVersionHash"}) private long checkedVersionHash; - @SerializedName(value = "isConsistent") + @SerializedName(value = "ic", alternate = {"isConsistent"}) private boolean isConsistent; // cooldown conf - @SerializedName(value = "cooldownReplicaId") + @SerializedName(value = "cri", alternate = {"cooldownReplicaId"}) private long cooldownReplicaId = -1; - @SerializedName(value = "cooldownTerm") + @SerializedName(value = "ctm", alternate = {"cooldownTerm"}) private long cooldownTerm = -1; private ReentrantReadWriteLock cooldownConfLock = new ReentrantReadWriteLock(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
