[CARBONDATA-1935]fix the backword compatibility issue for tableInfo deserialization
This closes #1720 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/03ddcc85 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/03ddcc85 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/03ddcc85 Branch: refs/heads/branch-1.3 Commit: 03ddcc85db37cba37aedd3feb2f99f43d14b0c8a Parents: be5134e Author: akashrn5 <[email protected]> Authored: Fri Dec 22 18:37:20 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Tue Jan 2 16:40:39 2018 +0800 ---------------------------------------------------------------------- .../core/metadata/datatype/DataTypeAdapter.java | 51 ++++++++++++++++++++ .../core/metadata/datatype/DataTypes.java | 16 +++--- .../apache/carbondata/core/util/CarbonUtil.java | 11 ++++- .../carbondata/core/util/DataTypeUtil.java | 40 +++++++++++++++ .../table/CarbonDescribeFormattedCommand.scala | 2 +- 5 files changed, 110 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java new file mode 100644 index 0000000..08fbcca --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.metadata.datatype; + +import java.io.IOException; + +import org.apache.carbondata.core.util.DataTypeUtil; + +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; + +/** + * This class is added to support backward compatibility with table info object, where DATATYPE + * is string in old version(1.2) and OBJECT in new version(1.3) + */ +public class DataTypeAdapter extends TypeAdapter<Object> { + + private Gson fallBack_original = new Gson(); + + @Override public void write(JsonWriter jsonWriter, Object o) throws IOException { + } + + @Override public Object read(JsonReader jsonReader) throws IOException { + JsonToken token = jsonReader.peek(); + if (token == JsonToken.STRING) { + return DataTypeUtil.valueOf(jsonReader.nextString()); + } else { + // use original deserializer logic + return fallBack_original.fromJson(jsonReader, DataType.class); + } + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java index e96917f..ad21eaa 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java @@ -58,14 +58,14 @@ public class DataTypes { static final int LONG_TYPE_ID = 7; static final int DOUBLE_TYPE_ID = 8; static final int NULL_TYPE_ID = 9; - static final int BYTE_TYPE_ID = 10; - static final int BYTE_ARRAY_TYPE_ID = 11; - static final int SHORT_INT_TYPE_ID = 12; - static final int LEGACY_LONG_TYPE_ID = 13; - static final int DECIMAL_TYPE_ID = 20; - static final int ARRAY_TYPE_ID = 21; - static final int STRUCT_TYPE_ID = 22; - static final int MAP_TYPE_ID = 23; + static final int BYTE_TYPE_ID = 14; + static final int BYTE_ARRAY_TYPE_ID = 15; + static final int SHORT_INT_TYPE_ID = 16; + static final int LEGACY_LONG_TYPE_ID = 17; + static final int DECIMAL_TYPE_ID = 10; + static final int ARRAY_TYPE_ID = 11; + static final int STRUCT_TYPE_ID = 12; + static final int MAP_TYPE_ID = 13; /** * create a DataType instance from uniqueId of the DataType http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index bd01772..ffe4654 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -65,6 +65,7 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; import org.apache.carbondata.core.metadata.blocklet.SegmentInfo; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypeAdapter; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema; @@ -90,6 +91,8 @@ import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configuration; @@ -1924,7 +1927,6 @@ public final class CarbonUtil { // TODO: move this to carbon store API as it is related to TableInfo creation public static TableInfo convertGsonToTableInfo(Map<String, String> properties) { - Gson gson = new Gson(); String partsNo = properties.get("carbonSchemaPartsNo"); if (partsNo == null) { return null; @@ -1938,6 +1940,13 @@ public final class CarbonUtil { } builder.append(part); } + + // Datatype GSON adapter is added to support backward compatibility for tableInfo + // deserialization + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(DataType.class, new DataTypeAdapter()); + + Gson gson = gsonBuilder.create(); TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class); // The tableInfo is deserialized from GSON string, need to update the scale and http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 0e819c4..65ea63e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -735,4 +735,44 @@ public final class DataTypeUtil { return converter; } + /** + * @param name datatype string extracted from the json data + * @return returns the datatype based on the input string from json to deserialize the tableInfo + */ + public static DataType valueOf(String name) { + if (DataTypes.STRING.getName().equalsIgnoreCase(name)) { + return DataTypes.STRING; + } else if (DataTypes.DATE.getName().equalsIgnoreCase(name)) { + return DataTypes.DATE; + } else if (DataTypes.TIMESTAMP.getName().equalsIgnoreCase(name)) { + return DataTypes.TIMESTAMP; + } else if (DataTypes.BOOLEAN.getName().equalsIgnoreCase(name)) { + return DataTypes.BOOLEAN; + } else if (DataTypes.BYTE.getName().equalsIgnoreCase(name)) { + return DataTypes.BYTE; + } else if (DataTypes.SHORT.getName().equalsIgnoreCase(name)) { + return DataTypes.SHORT; + } else if (DataTypes.SHORT_INT.getName().equalsIgnoreCase(name)) { + return DataTypes.SHORT_INT; + } else if (DataTypes.INT.getName().equalsIgnoreCase(name)) { + return DataTypes.INT; + } else if (DataTypes.LONG.getName().equalsIgnoreCase(name)) { + return DataTypes.LONG; + } else if (DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(name)) { + return DataTypes.LEGACY_LONG; + } else if (DataTypes.FLOAT.getName().equalsIgnoreCase(name)) { + return DataTypes.FLOAT; + } else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(name)) { + return DataTypes.DOUBLE; + } else if (DataTypes.NULL.getName().equalsIgnoreCase(name)) { + return DataTypes.NULL; + } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) { + return DataTypes.BYTE_ARRAY; + } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) { + return DataTypes.BYTE_ARRAY; + } else { + throw new RuntimeException("create DataType with invalid name: " + name); + } + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala index 5d19e0a..41656e5 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala @@ -109,7 +109,7 @@ private[sql] case class CarbonDescribeFormattedCommand( results ++= Seq(("Streaming", isStreaming, "")) val tblProps = carbonTable.getTableInfo.getFactTable.getTableProperties - results ++= Seq(("SORT_SCOPE", tblProps.getOrDefault("sort_scope", CarbonCommonConstants + results ++= Seq(("SORT_SCOPE", tblProps.asScala.getOrElse("sort_scope", CarbonCommonConstants .LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)) // show table level compaction options
