This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch fix_195_chinese_ChunkGroupMetadata in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 07c36997631f2f339c0f8045192b9a09c02e4af7 Author: xiangdong huang <[email protected]> AuthorDate: Wed Sep 18 14:27:08 2019 +0800 using String.getBytes(utf-9).length to replace string.length() in ChunkGroupMetadata for supporting Chinese --- .../java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 2 +- .../org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 2 ++ .../apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java | 2 +- .../org/apache/iotdb/tsfile/file/header/ChunkHeader.java | 7 +------ .../iotdb/tsfile/file/metadata/ChunkGroupMetaData.java | 12 ++++++++---- .../apache/iotdb/tsfile/file/metadata/ChunkMetaData.java | 8 +------- .../main/java/org/apache/iotdb/tsfile/utils/Binary.java | 10 +++------- .../java/org/apache/iotdb/tsfile/utils/BytesUtils.java | 14 ++------------ .../org/apache/iotdb/tsfile/utils/StringContainer.java | 1 + 9 files changed, 20 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java index ad42d18..d9addec 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java @@ -81,7 +81,7 @@ public class IoTDBConfigCheck { // get existed properties from system_properties.txt File inputFile = SystemFileFactory.INSTANCE.getFile(filepath + File.separator + PROPERTIES_FILE_NAME); try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) { - properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING)); + properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)); if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) { logger.error("Wrong timestamp precision, please set as: " + properties .getProperty("timestamp_precision") + " !"); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 79b6622..39affd9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.common.conf; +import java.nio.charset.Charset; import org.apache.iotdb.tsfile.fileSystem.FSType; /** @@ -49,6 +50,7 @@ public class TSFileConfig { */ public static final int BYTE_SIZE_PER_CHAR = 4; public static final String STRING_ENCODING = "UTF-8"; + public static final Charset STRING_CHARSET = Charset.forName(STRING_ENCODING); public static final String CONFIG_FILE_NAME = "tsfile-format.properties"; public static final String MAGIC_STRING = "TsFilev0.8.0"; /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java index e440956..e1fa0b7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/footer/ChunkGroupFooter.java @@ -55,7 +55,7 @@ public class ChunkGroupFooter { this.dataSize = dataSize; this.numberOfChunks = numberOfChunks; this.serializedSize = - Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + Long.BYTES + Integer.BYTES; + Byte.BYTES + Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Long.BYTES + Integer.BYTES; } public static int getSerializedSize(String deviceID) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java index 0dc5ef0..b2e7279f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java @@ -76,12 +76,7 @@ public class ChunkHeader { } public static int getSerializedSize(String measurementID) { - try { - return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_ENCODING).length); - } catch (UnsupportedEncodingException e) { - LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING); - return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes().length); - } + return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_CHARSET).length); } private static int getSerializedSize(int measurementIdLength) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java index a717eba..c81d074 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.file.metadata; +import java.io.UnsupportedEncodingException; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -28,11 +29,14 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Metadata of ChunkGroup. */ public class ChunkGroupMetaData { + private static final Logger logger = LoggerFactory.getLogger(ChunkGroupMetaData.class); /** * Name of device, this field is not serialized. @@ -104,7 +108,7 @@ public class ChunkGroupMetaData { int size = ReadWriteIOUtils.readInt(inputStream); chunkGroupMetaData.serializedSize = Integer.BYTES - + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES; List<ChunkMetaData> chunkMetaDataList = new ArrayList<>(); @@ -135,7 +139,7 @@ public class ChunkGroupMetaData { int size = ReadWriteIOUtils.readInt(buffer); - chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_ENCODING).length + chunkGroupMetaData.serializedSize = Integer.BYTES + chunkGroupMetaData.deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES + Long.BYTES + Long.BYTES + Long.BYTES; List<ChunkMetaData> chunkMetaDataList = new ArrayList<>(); @@ -154,8 +158,8 @@ public class ChunkGroupMetaData { } void reCalculateSerializedSize() { - serializedSize = Integer.BYTES + deviceID.length() + Integer.BYTES - + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList + serializedSize = Integer.BYTES + deviceID.getBytes(TSFileConfig.STRING_CHARSET).length + Integer.BYTES + + Long.BYTES + Long.BYTES + Long.BYTES; // size of chunkMetaDataList for (ChunkMetaData chunk : chunkMetaDataList) { serializedSize += chunk.getSerializedSize(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java index 269e861..c94c654 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java @@ -146,13 +146,7 @@ public class ChunkMetaData { TSDataType.getSerializedSize() + // TSDataType (valuesStatistics == null ? TsDigest.getNullDigestSize() : valuesStatistics.getSerializedSize())); - try { - serializedSize += measurementUid.getBytes(TSFileConfig.STRING_ENCODING).length; // measurementUid - } catch (UnsupportedEncodingException e) { - // use the system default encoding - serializedSize += measurementUid.getBytes().length; // measurementUid - LOG.error("{} encoding is not supported", TSFileConfig.STRING_ENCODING); - } + serializedSize += measurementUid.getBytes(TSFileConfig.STRING_CHARSET).length; // measurementUid return serializedSize; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java index b071806..61d8323 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java @@ -34,7 +34,6 @@ public class Binary implements Comparable<Binary>, Serializable { private static final long serialVersionUID = 6394197743397020735L; private byte[] values; - private String textEncodingType = TSFileConfig.STRING_ENCODING; /** * if the bytes v is modified, the modification is visible to this binary. @@ -44,7 +43,7 @@ public class Binary implements Comparable<Binary>, Serializable { } public Binary(String s) { - this.values = (s == null) ? null : s.getBytes(Charset.forName(this.textEncodingType)); + this.values = (s == null) ? null : s.getBytes(TSFileConfig.STRING_CHARSET); } public static Binary valueOf(String value) { @@ -105,16 +104,13 @@ public class Binary implements Comparable<Binary>, Serializable { } public String getStringValue() { - return new String(this.values, Charset.forName(this.textEncodingType)); + return new String(this.values, TSFileConfig.STRING_CHARSET); } public String getTextEncodingType() { - return textEncodingType; + return TSFileConfig.STRING_ENCODING; } - public void setTextEncodingType(String textEncodingType) { - this.textEncodingType = textEncodingType; - } @Override public String toString() { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java index 990dba9..20c6a58 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java @@ -567,12 +567,7 @@ public class BytesUtils { * @return byte array */ public static byte[] stringToBytes(String str) { - try { - return str.getBytes(TSFileConfig.STRING_ENCODING); - } catch (UnsupportedEncodingException e) { - LOG.error("catch UnsupportedEncodingException {}", str, e); - return null; - } + return str.getBytes(TSFileConfig.STRING_CHARSET); } /** @@ -582,12 +577,7 @@ public class BytesUtils { * @return string */ public static String bytesToString(byte[] byteStr) { - try { - return new String(byteStr, TSFileConfig.STRING_ENCODING); - } catch (UnsupportedEncodingException e) { - LOG.error("catch UnsupportedEncodingException {}", byteStr, e); - return null; - } + return new String(byteStr, TSFileConfig.STRING_CHARSET); } /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java index 8212db1..bbe2179 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/StringContainer.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.utils; import java.util.ArrayList; import java.util.List; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; /** * this class is used to contact String effectively.It contains a StringBuider and initialize it
