This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 732e37d7da089d5156bc91a3097152999469291c Author: Yongqiang YANG <[email protected]> AuthorDate: Fri Jul 5 10:02:17 2024 +0800 [fix](partitionkey) serialize expr in PartitionKey via gson (#37277) --- .../org/apache/doris/catalog/PrimitiveType.java | 7 ++ .../org/apache/doris/common/FeMetaVersion.java | 5 +- .../main/java/org/apache/doris/catalog/Index.java | 12 +-- .../doris/catalog/MaterializedIndexMeta.java | 20 ++-- .../java/org/apache/doris/catalog/Partition.java | 24 ++--- .../org/apache/doris/catalog/PartitionKey.java | 113 ++++++++++----------- 6 files changed, 95 insertions(+), 86 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 50cc6c4c1c0..e5b1b9c958b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -23,11 +23,13 @@ import org.apache.doris.thrift.TPrimitiveType; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.util.ArrayList; import java.util.List; public enum PrimitiveType { + // DO NOT CHANGE desc and to string of these types, they are used in persist INVALID_TYPE("INVALID_TYPE", -1, TPrimitiveType.INVALID_TYPE, false), UNSUPPORTED("UNSUPPORTED_TYPE", -1, TPrimitiveType.UNSUPPORTED, false), // NULL_TYPE - used only in LiteralPredicate and NullLiteral to make NULLs compatible @@ -699,10 +701,15 @@ public enum PrimitiveType { return implicitCastMap.get(type).contains(target); } + @SerializedName("d") private final String description; + @SerializedName("s") private final int slotSize; // size of tuple slot for this type + @SerializedName("t") private final TPrimitiveType thriftType; + @SerializedName("a") private final boolean availableInDdl; + @SerializedName("it") private boolean isTimeType = false; PrimitiveType(String description, int slotSize, TPrimitiveType thriftType, boolean availableInDdl) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 4f2be870987..a81a01227e7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -101,8 +101,11 @@ public final class FeMetaVersion { public static final int VERSION_139 = 139; + public static final int VERSION_140 = 140; + // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_139; + public static final int VERSION_CURRENT = VERSION_140; + // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 5b0c9976819..52290bc7479 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -51,17 +51,17 @@ import java.util.Set; public class Index implements Writable { public static final int INDEX_ID_INIT_VALUE = -1; - @SerializedName(value = "indexId") + @SerializedName(value = "i", alternate = {"indexId"}) private long indexId = -1; // -1 for compatibale - @SerializedName(value = "indexName") + @SerializedName(value = "in", alternate = {"indexName"}) private String indexName; - @SerializedName(value = "columns") + @SerializedName(value = "c", alternate = {"columns"}) private List<String> columns; - @SerializedName(value = "indexType") + @SerializedName(value = "it", alternate = {"indexType"}) private IndexDef.IndexType indexType; - @SerializedName(value = "properties") + @SerializedName(value = "pt", alternate = {"properties"}) private Map<String, String> properties; - @SerializedName(value = "comment") + @SerializedName(value = "ct", alternate = {"comment"}) private String comment; public Index(long indexId, String indexName, List<String> columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 79274b3bb0b..0069d4e4e02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -49,26 +49,26 @@ import java.util.Map; import java.util.stream.Collectors; public class MaterializedIndexMeta implements Writable, GsonPostProcessable { - @SerializedName(value = "indexId") + @SerializedName(value = "id", alternate = {"indexId"}) private long indexId; - @SerializedName(value = "schema") + @SerializedName(value = "sc", alternate = {"schema"}) private List<Column> schema = Lists.newArrayList(); - @SerializedName(value = "schemaVersion") + @SerializedName(value = "sv", alternate = {"schemaVersion"}) private int schemaVersion; - @SerializedName(value = "schemaHash") + @SerializedName(value = "sh", alternate = {"schemaHash"}) private int schemaHash; - @SerializedName(value = "shortKeyColumnCount") + @SerializedName(value = "skcc", alternate = {"shortKeyColumnCount"}) private short shortKeyColumnCount; - @SerializedName(value = "storageType") + @SerializedName(value = "st", alternate = {"storageType"}) private TStorageType storageType; - @SerializedName(value = "keysType") + @SerializedName(value = "kt", alternate = {"keysType"}) private KeysType keysType; - @SerializedName(value = "defineStmt") + @SerializedName(value = "dst", alternate = {"defineStmt"}) private OriginStatement defineStmt; //for light schema change - @SerializedName(value = "maxColUniqueId") + @SerializedName(value = "mcui", alternate = {"maxColUniqueId"}) private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE; - @SerializedName(value = "indexes") + @SerializedName(value = "idx", alternate = {"indexes"}) private List<Index> indexes; private Expr whereClause; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java index ef94da902b8..c28d44a5521 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java @@ -60,24 +60,24 @@ public class Partition extends MetaObject { @SerializedName(value = "id") private long id; - @SerializedName(value = "name") + @SerializedName(value = "nm", alternate = {"name"}) private String name; - @SerializedName(value = "state") + @SerializedName(value = "st", alternate = {"state"}) private PartitionState state; - @SerializedName(value = "baseIndex") + @SerializedName(value = "bi", alternate = {"baseIndex"}) private MaterializedIndex baseIndex; /** * Visible rollup indexes are indexes which are visible to user. * User can do query on them, show them in related 'show' stmt. */ - @SerializedName(value = "idToVisibleRollupIndex") + @SerializedName(value = "ivr", alternate = {"idToVisibleRollupIndex"}) private Map<Long, MaterializedIndex> idToVisibleRollupIndex = Maps.newHashMap(); /** * Shadow indexes are indexes which are not visible to user. * Query will not run on these shadow indexes, and user can not see them neither. * But load process will load data into these shadow indexes. */ - @SerializedName(value = "idToShadowIndex") + @SerializedName(value = "isi", alternate = {"idToShadowIndex"}) private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap(); /** @@ -88,21 +88,21 @@ public class Partition extends MetaObject { // not have committedVersion because committedVersion = nextVersion - 1 @Deprecated - @SerializedName(value = "committedVersionHash") + @SerializedName(value = "cvh", alternate = {"committedVersionHash"}) private long committedVersionHash; - @SerializedName(value = "visibleVersion") + @SerializedName(value = "vv", alternate = {"visibleVersion"}) private long visibleVersion; - @SerializedName(value = "visibleVersionTime") + @SerializedName(value = "vvt", alternate = {"visibleVersionTime"}) private long visibleVersionTime; @Deprecated - @SerializedName(value = "visibleVersionHash") + @SerializedName(value = "vvh", alternate = {"visibleVersionHash"}) private long visibleVersionHash; - @SerializedName(value = "nextVersion") + @SerializedName(value = "nv", alternate = {"nextVersion"}) protected long nextVersion; @Deprecated - @SerializedName(value = "nextVersionHash") + @SerializedName(value = "nvh", alternate = {"nextVersionHash"}) private long nextVersionHash; - @SerializedName(value = "distributionInfo") + @SerializedName(value = "di", alternate = {"distributionInfo"}) private DistributionInfo distributionInfo; protected Partition() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 31160dfeef1..1f38ce045ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -64,7 +64,6 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { private static final Logger LOG = LogManager.getLogger(PartitionKey.class); @SerializedName("ks") private List<LiteralExpr> keys; - @SerializedName("hk") private List<String> originHiveKeys; @SerializedName("ts") private List<PrimitiveType> types; @@ -511,11 +510,11 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { // added by ccr, and we have to follow. public static class PartitionKeySerializer implements JsonSerializer<PartitionKey>, JsonDeserializer<PartitionKey> { + @Override public JsonElement serialize(PartitionKey partitionKey, java.lang.reflect.Type reflectType, JsonSerializationContext context) { - JsonArray result = new JsonArray(); - + // for compatibility List<PrimitiveType> types = partitionKey.getTypes(); List<LiteralExpr> keys = partitionKey.getKeys(); int count = keys.size(); @@ -523,68 +522,68 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { throw new JsonParseException("Size of keys and types are not equal"); } + JsonArray jsonArray = new JsonArray(); for (int i = 0; i < count; i++) { JsonArray typeAndKey = new JsonArray(); - PrimitiveType type = types.get(i); - if (keys.get(i).isNullLiteral()) { - // save NULL_TYPE as type and real type as key - typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString())); - typeAndKey.add(new JsonPrimitive(type.toString())); - } else { - typeAndKey.add(new JsonPrimitive(type.toString())); - - if (keys.get(i) == MaxLiteral.MAX_VALUE) { - typeAndKey.add(new JsonPrimitive("MAX_VALUE")); - } else { - switch (type) { - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: { - IntLiteral key = (IntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getLongValue())); - } - break; - case LARGEINT: { - LargeIntLiteral key = (LargeIntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getRealValue().toString())); - } - break; - case DATE: - case DATETIME: - case DATEV2: - case DATETIMEV2: { - DateLiteral key = (DateLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.convertToString(type))); - } - break; - case CHAR: - case VARCHAR: - case STRING: { - StringLiteral key = (StringLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); - } - break; - case BOOLEAN: { - BoolLiteral key = (BoolLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); - } - break; - default: - throw new JsonParseException( - "type[" + type.name() + "] not supported: "); + typeAndKey.add(context.serialize(types.get(i))); + typeAndKey.add(context.serialize(keys.get(i))); + jsonArray.add(typeAndKey); + } + + // for compatibility in the future + jsonArray.add(new JsonPrimitive("unused")); + + return jsonArray; + } + + @Override + public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_140) { + return deserializeOld(json, typeOfT, context); + } else { + PartitionKey partitionKey = new PartitionKey(); + + JsonArray jsonArray = json.getAsJsonArray(); + for (int i = 0; i < jsonArray.size() - 1; i++) { + PrimitiveType type = null; + type = context.deserialize(jsonArray.get(i).getAsJsonArray().get(0), PrimitiveType.class); + LiteralExpr key = context.deserialize(jsonArray.get(i).getAsJsonArray().get(1), Expr.class); + + if (key instanceof NullLiteral) { + key = NullLiteral.create(Type.fromPrimitiveType(type)); + partitionKey.types.add(type); + partitionKey.keys.add(key); + continue; + } + if (key instanceof MaxLiteral) { + key = MaxLiteral.MAX_VALUE; + } + if (type != PrimitiveType.DATETIMEV2) { + key.setType(Type.fromPrimitiveType(type)); + } + if (type.isDateV2Type()) { + try { + key.checkValueValid(); + } catch (AnalysisException e) { + LOG.warn("Value {} for partition key [type = {}] is invalid! This is a bug exists " + + "in Doris 1.2.0 and fixed since Doris 1.2.1. You should create this table " + + "again using Doris 1.2.1+ .", key.getStringValue(), type); + ((DateLiteral) key).setMinValue(); } } + + partitionKey.types.add(type); + partitionKey.keys.add(key); } - result.add(typeAndKey); + // ignore the last element + return partitionKey; } - - return result; } - @Override - public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT, + // can be removed after 3.0.0 + private PartitionKey deserializeOld(JsonElement json, java.lang.reflect.Type typeOfT, JsonDeserializationContext context) throws JsonParseException { PartitionKey partitionKey = new PartitionKey(); JsonArray jsonArray = json.getAsJsonArray(); @@ -627,7 +626,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { case DATETIMEV2: { String value = typeAndKey.get(1).getAsString(); try { - literal = new DateLiteral(value); + literal = new DateLiteral(value, Type.fromPrimitiveType(type)); } catch (AnalysisException e) { throw new JsonParseException("DateLiteral deserialize failed: " + e.getMessage()); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
