Alima777 commented on a change in pull request #2246: URL: https://github.com/apache/iotdb/pull/2246#discussion_r540909455
########## File path: server/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.PartialPath; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.TSRecord; +import org.apache.iotdb.tsfile.write.record.datapoint.*; +import org.junit.Assert; +import org.junit.Test; + +public class MemUtilsTest { + + @Test + public void recordSizeTest() throws IllegalPathException { + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.INT32, 10, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.INT64, 10, true)); + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.FLOAT, 10.0, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.DOUBLE, 10.0, true)); + Assert.assertEquals(8, MemUtils.getRecordSize(TSDataType.TEXT, "10", false)); + + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2", "s3", "s4", "s5"}; + List<Integer> dataTypes = new ArrayList<>(); + int sizeSum = 0; + dataTypes.add(TSDataType.INT32.ordinal()); + sizeSum += 8 + TSDataType.INT32.getDataTypeSize(); + dataTypes.add(TSDataType.INT64.ordinal()); + sizeSum += 8 + TSDataType.INT64.getDataTypeSize(); + dataTypes.add(TSDataType.FLOAT.ordinal()); + sizeSum += 8 + TSDataType.FLOAT.getDataTypeSize(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + sizeSum += 8 + TSDataType.DOUBLE.getDataTypeSize(); + dataTypes.add(TSDataType.TEXT.ordinal()); + sizeSum += TSDataType.TEXT.getDataTypeSize(); + InsertTabletPlan insertPlan = new InsertTabletPlan(device, measurements, dataTypes); + Assert.assertEquals(sizeSum, MemUtils.getRecordSize(insertPlan, 0, 1, false)); + } + + @Test + public void TsRecordSizeTest() { Review comment: `TsRecordSizeTest` -> `TsRecordMemTest`. And add javadoc for this test, it tests getDataPointMem() and getTsRecordMem(). ########## File path: server/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.PartialPath; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.TSRecord; +import org.apache.iotdb.tsfile.write.record.datapoint.*; +import org.junit.Assert; +import org.junit.Test; + +public class MemUtilsTest { + + @Test + public void recordSizeTest() throws IllegalPathException { + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.INT32, 10, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.INT64, 10, true)); + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.FLOAT, 10.0, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.DOUBLE, 10.0, true)); + Assert.assertEquals(8, MemUtils.getRecordSize(TSDataType.TEXT, "10", false)); + + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2", "s3", "s4", "s5"}; + List<Integer> dataTypes = new ArrayList<>(); + int sizeSum = 0; + dataTypes.add(TSDataType.INT32.ordinal()); + sizeSum += 8 + TSDataType.INT32.getDataTypeSize(); + dataTypes.add(TSDataType.INT64.ordinal()); + sizeSum += 8 + TSDataType.INT64.getDataTypeSize(); + dataTypes.add(TSDataType.FLOAT.ordinal()); + sizeSum += 8 + TSDataType.FLOAT.getDataTypeSize(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + sizeSum += 8 + TSDataType.DOUBLE.getDataTypeSize(); + dataTypes.add(TSDataType.TEXT.ordinal()); + sizeSum += TSDataType.TEXT.getDataTypeSize(); + InsertTabletPlan insertPlan = new InsertTabletPlan(device, measurements, dataTypes); + Assert.assertEquals(sizeSum, MemUtils.getRecordSize(insertPlan, 0, 1, false)); + } Review comment: What about seperating this single test to two? I think it's better to keep the test atomic. ########## File path: server/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TimeValuePair; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +public class SerializeUtilsTest { + @Test + public void serdesStringTest() { + String str = "abcd%+/123\n\t"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(str, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(str, SerializeUtils.deserializeString(buffer)); + } + + @Test + public void serdesStringListTest() { + List<String> slist = Arrays.asList("abc", "123", "y87@", "9+&d\n"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeStringList(slist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(slist, SerializeUtils.deserializeStringList(buffer)); + } + + @Test + public void serdesIntListTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intlist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesIntSetTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + Set<Integer> intSet = new TreeSet<>(intlist); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intSet, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesNodeTest() { + Node node = new Node("127.0.0.1", 6667, 1, 6535, 4678); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(node, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Node anotherNode = new Node("127.0.0.1", 6667, 1, 6535, 4678); + SerializeUtils.deserialize(anotherNode, buffer); + Assert.assertEquals(node, anotherNode); + } + + @Test + public void serdesINT32BatchDataTest() { + BatchData batchData = new BatchData(TSDataType.INT32); + int ivalue = 0; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, ivalue); + ivalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesINT64BatchDataTest() { + BatchData batchData = new BatchData(TSDataType.INT64); + long lvalue = 0; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, lvalue); + lvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesFLOATBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.FLOAT); + float fvalue = 0f; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, fvalue); + fvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesDOUBLEBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.DOUBLE); + double dvalue = 0d; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, dvalue); + dvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesBOOLEANBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.BOOLEAN); + batchData.putAnObject(1, true); + batchData.putAnObject(2, false); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesTEXTBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.TEXT); + String svalue = ""; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, Binary.valueOf(svalue)); + svalue += String.valueOf(time); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesTVPairTest() { + List<TimeValuePair> TVPairs = new ArrayList<>(); + TimeValuePair p1 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.BOOLEAN, true)); + TVPairs.add(p1); Review comment: Add javadoc for this and the next test. ########## File path: server/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TimeValuePair; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +public class SerializeUtilsTest { + @Test + public void serdesStringTest() { Review comment: I don't think `serdes` is a good abbreviation... ########## File path: server/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TimeValuePair; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +public class SerializeUtilsTest { + @Test + public void serdesStringTest() { + String str = "abcd%+/123\n\t"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(str, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(str, SerializeUtils.deserializeString(buffer)); + } + + @Test + public void serdesStringListTest() { + List<String> slist = Arrays.asList("abc", "123", "y87@", "9+&d\n"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeStringList(slist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(slist, SerializeUtils.deserializeStringList(buffer)); + } + + @Test + public void serdesIntListTest() { Review comment: Hi, I checked the name of this method in SerializeUtils, which called just serialize. FMP, it's better to rename it. `serialize` -> `serializeIntList`, `serialize` -> `serializeIntSet`... ########## File path: server/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.PartialPath; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.TSRecord; +import org.apache.iotdb.tsfile.write.record.datapoint.*; +import org.junit.Assert; +import org.junit.Test; + +public class MemUtilsTest { + + @Test + public void recordSizeTest() throws IllegalPathException { + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.INT32, 10, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.INT64, 10, true)); + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.FLOAT, 10.0, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.DOUBLE, 10.0, true)); + Assert.assertEquals(8, MemUtils.getRecordSize(TSDataType.TEXT, "10", false)); + + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2", "s3", "s4", "s5"}; + List<Integer> dataTypes = new ArrayList<>(); + int sizeSum = 0; + dataTypes.add(TSDataType.INT32.ordinal()); + sizeSum += 8 + TSDataType.INT32.getDataTypeSize(); + dataTypes.add(TSDataType.INT64.ordinal()); + sizeSum += 8 + TSDataType.INT64.getDataTypeSize(); + dataTypes.add(TSDataType.FLOAT.ordinal()); + sizeSum += 8 + TSDataType.FLOAT.getDataTypeSize(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + sizeSum += 8 + TSDataType.DOUBLE.getDataTypeSize(); + dataTypes.add(TSDataType.TEXT.ordinal()); + sizeSum += TSDataType.TEXT.getDataTypeSize(); + InsertTabletPlan insertPlan = new InsertTabletPlan(device, measurements, dataTypes); + Assert.assertEquals(sizeSum, MemUtils.getRecordSize(insertPlan, 0, 1, false)); + } + + @Test + public void TsRecordSizeTest() { + int totalSize = 0; + String device = "root.sg.d1"; + TSRecord record = new TSRecord(0, device); + + DataPoint point1 = new IntDataPoint("s1", 1); + Assert.assertEquals(MemUtils.getStringMem("s1") + 20, MemUtils.getDataPointMem(point1)); + totalSize += MemUtils.getDataPointMem(point1); + record.addTuple(point1); + + DataPoint point2 = new LongDataPoint("s2", 1); + Assert.assertEquals(MemUtils.getStringMem("s2") + 24, MemUtils.getDataPointMem(point2)); + totalSize += MemUtils.getDataPointMem(point2); + record.addTuple(point2); + + DataPoint point3 = new FloatDataPoint("s3", 1.0f); + Assert.assertEquals(MemUtils.getStringMem("s3") + 20, MemUtils.getDataPointMem(point3)); + totalSize += MemUtils.getDataPointMem(point3); + record.addTuple(point3); + + DataPoint point4 = new DoubleDataPoint("s4", 1.0d); + Assert.assertEquals(MemUtils.getStringMem("s4") + 24, MemUtils.getDataPointMem(point4)); + totalSize += MemUtils.getDataPointMem(point4); + record.addTuple(point4); + + DataPoint point5 = new BooleanDataPoint("s5", true); + Assert.assertEquals(MemUtils.getStringMem("s5") + 17, MemUtils.getDataPointMem(point5)); + totalSize += MemUtils.getDataPointMem(point5); + record.addTuple(point5); + + totalSize += 8 * record.dataPointList.size() + MemUtils.getStringMem(device)+ 16; Review comment: ```suggestion totalSize += 8 * record.dataPointList.size() + MemUtils.getStringMem(device) + 16; ``` ########## File path: server/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TimeValuePair; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +public class SerializeUtilsTest { + @Test + public void serdesStringTest() { + String str = "abcd%+/123\n\t"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(str, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(str, SerializeUtils.deserializeString(buffer)); + } + + @Test + public void serdesStringListTest() { + List<String> slist = Arrays.asList("abc", "123", "y87@", "9+&d\n"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeStringList(slist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(slist, SerializeUtils.deserializeStringList(buffer)); + } + + @Test + public void serdesIntListTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intlist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesIntSetTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + Set<Integer> intSet = new TreeSet<>(intlist); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intSet, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesNodeTest() { + Node node = new Node("127.0.0.1", 6667, 1, 6535, 4678); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(node, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Node anotherNode = new Node("127.0.0.1", 6667, 1, 6535, 4678); + SerializeUtils.deserialize(anotherNode, buffer); + Assert.assertEquals(node, anotherNode); + } + + @Test + public void serdesINT32BatchDataTest() { + BatchData batchData = new BatchData(TSDataType.INT32); + int ivalue = 0; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, ivalue); + ivalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesINT64BatchDataTest() { + BatchData batchData = new BatchData(TSDataType.INT64); + long lvalue = 0; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, lvalue); + lvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesFLOATBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.FLOAT); + float fvalue = 0f; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, fvalue); + fvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesDOUBLEBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.DOUBLE); + double dvalue = 0d; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, dvalue); + dvalue++; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesBOOLEANBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.BOOLEAN); + batchData.putAnObject(1, true); + batchData.putAnObject(2, false); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesTEXTBatchDataTest() { + BatchData batchData = new BatchData(TSDataType.TEXT); + String svalue = ""; + for (long time = 0; time < 10; time++) { + batchData.putAnObject(time, Binary.valueOf(svalue)); + svalue += String.valueOf(time); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeBatchData(batchData, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + BatchData anotherBatch = SerializeUtils.deserializeBatchData(buffer); + while (batchData.hasCurrent()) { + Assert.assertEquals(batchData.currentValue(), anotherBatch.currentValue()); + batchData.next(); + anotherBatch.next(); + } + } + + @Test + public void serdesTVPairTest() { + List<TimeValuePair> TVPairs = new ArrayList<>(); + TimeValuePair p1 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.BOOLEAN, true)); + TVPairs.add(p1); + TimeValuePair p2 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT32, 1)); + TVPairs.add(p2); + TimeValuePair p3 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT64, 1L)); + TVPairs.add(p3); + TimeValuePair p4 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.FLOAT, 1.0f)); + TVPairs.add(p4); + TimeValuePair p5 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.DOUBLE, 1.0d)); + TVPairs.add(p5); + TimeValuePair p6 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.TEXT, Binary.valueOf("a"))); + TVPairs.add(p6); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + for (TimeValuePair tv : TVPairs) { + SerializeUtils.serializeTVPair(tv, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(tv, SerializeUtils.deserializeTVPair(buffer)); + baos.reset(); + } + + } + + @Test + public void serdesTVPairsTest() { + List<List<TimeValuePair>> TVPairs = new ArrayList<>(); + TimeValuePair p1 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.BOOLEAN, true)); + TVPairs.add(Collections.singletonList(p1)); + TimeValuePair p2 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT32, 1)); + TVPairs.add(Collections.singletonList(p2)); + TimeValuePair p3 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.INT64, 1L)); + TVPairs.add(Collections.singletonList(p3)); + TimeValuePair p4 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.FLOAT, 1.0f)); + TVPairs.add(Collections.singletonList(p4)); + TimeValuePair p5 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.DOUBLE, 1.0d)); + TVPairs.add(Collections.singletonList(p5)); + TimeValuePair p6 = new TimeValuePair(0, TsPrimitiveType.getByType(TSDataType.TEXT, Binary.valueOf("a"))); + TVPairs.add(Collections.singletonList(p6)); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + for (List<TimeValuePair> tv : TVPairs) { + SerializeUtils.serializeTVPairs(tv, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(tv, SerializeUtils.deserializeTVPairs(buffer)); + baos.reset(); + } + } + + @Test + public void serdesObjectTest() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeObject(1, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(1, SerializeUtils.deserializeObject(buffer)); + } + + @Test + public void serdesObjectsTest() { + Object[] objects = { 1, "2", 3d}; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeObjects(objects, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertArrayEquals(objects, SerializeUtils.deserializeObjects(buffer)); + } + + @Test + public void serdesLongTest() { + long[] array = {1, 10, 100, 1000, 10000}; Review comment: It's Longs here. And add javadoc. ########## File path: server/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.PartialPath; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.TSRecord; +import org.apache.iotdb.tsfile.write.record.datapoint.*; +import org.junit.Assert; +import org.junit.Test; + +public class MemUtilsTest { + + @Test + public void recordSizeTest() throws IllegalPathException { + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.INT32, 10, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.INT64, 10, true)); + Assert.assertEquals(12, MemUtils.getRecordSize(TSDataType.FLOAT, 10.0, true)); + Assert.assertEquals(16, MemUtils.getRecordSize(TSDataType.DOUBLE, 10.0, true)); + Assert.assertEquals(8, MemUtils.getRecordSize(TSDataType.TEXT, "10", false)); + + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2", "s3", "s4", "s5"}; + List<Integer> dataTypes = new ArrayList<>(); + int sizeSum = 0; + dataTypes.add(TSDataType.INT32.ordinal()); + sizeSum += 8 + TSDataType.INT32.getDataTypeSize(); + dataTypes.add(TSDataType.INT64.ordinal()); + sizeSum += 8 + TSDataType.INT64.getDataTypeSize(); + dataTypes.add(TSDataType.FLOAT.ordinal()); + sizeSum += 8 + TSDataType.FLOAT.getDataTypeSize(); + dataTypes.add(TSDataType.DOUBLE.ordinal()); + sizeSum += 8 + TSDataType.DOUBLE.getDataTypeSize(); + dataTypes.add(TSDataType.TEXT.ordinal()); + sizeSum += TSDataType.TEXT.getDataTypeSize(); + InsertTabletPlan insertPlan = new InsertTabletPlan(device, measurements, dataTypes); + Assert.assertEquals(sizeSum, MemUtils.getRecordSize(insertPlan, 0, 1, false)); + } + + @Test + public void TsRecordSizeTest() { + int totalSize = 0; + String device = "root.sg.d1"; + TSRecord record = new TSRecord(0, device); + + DataPoint point1 = new IntDataPoint("s1", 1); + Assert.assertEquals(MemUtils.getStringMem("s1") + 20, MemUtils.getDataPointMem(point1)); + totalSize += MemUtils.getDataPointMem(point1); + record.addTuple(point1); + + DataPoint point2 = new LongDataPoint("s2", 1); + Assert.assertEquals(MemUtils.getStringMem("s2") + 24, MemUtils.getDataPointMem(point2)); + totalSize += MemUtils.getDataPointMem(point2); + record.addTuple(point2); + + DataPoint point3 = new FloatDataPoint("s3", 1.0f); + Assert.assertEquals(MemUtils.getStringMem("s3") + 20, MemUtils.getDataPointMem(point3)); + totalSize += MemUtils.getDataPointMem(point3); + record.addTuple(point3); + + DataPoint point4 = new DoubleDataPoint("s4", 1.0d); + Assert.assertEquals(MemUtils.getStringMem("s4") + 24, MemUtils.getDataPointMem(point4)); + totalSize += MemUtils.getDataPointMem(point4); + record.addTuple(point4); + + DataPoint point5 = new BooleanDataPoint("s5", true); + Assert.assertEquals(MemUtils.getStringMem("s5") + 17, MemUtils.getDataPointMem(point5)); + totalSize += MemUtils.getDataPointMem(point5); + record.addTuple(point5); Review comment: Why not check StringDataPoint? ########## File path: server/src/test/java/org/apache/iotdb/db/utils/SerializeUtilsTest.java ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TimeValuePair; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.junit.Assert; +import org.junit.Test; + +public class SerializeUtilsTest { + @Test + public void serdesStringTest() { + String str = "abcd%+/123\n\t"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(str, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(str, SerializeUtils.deserializeString(buffer)); + } + + @Test + public void serdesStringListTest() { + List<String> slist = Arrays.asList("abc", "123", "y87@", "9+&d\n"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serializeStringList(slist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Assert.assertEquals(slist, SerializeUtils.deserializeStringList(buffer)); + } + + @Test + public void serdesIntListTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intlist, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesIntSetTest() { + List<Integer> intlist = Arrays.asList(12, 34, 567, 8910); + Set<Integer> intSet = new TreeSet<>(intlist); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(intSet, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + List<Integer> anotherIntlist = new ArrayList<>(); + SerializeUtils.deserialize(anotherIntlist, buffer); + Assert.assertEquals(intlist, anotherIntlist); + } + + @Test + public void serdesNodeTest() { + Node node = new Node("127.0.0.1", 6667, 1, 6535, 4678); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(baos); + SerializeUtils.serialize(node, outputStream); + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Node anotherNode = new Node("127.0.0.1", 6667, 1, 6535, 4678); + SerializeUtils.deserialize(anotherNode, buffer); + Assert.assertEquals(node, anotherNode); + } + + @Test + public void serdesINT32BatchDataTest() { + BatchData batchData = new BatchData(TSDataType.INT32); + int ivalue = 0; Review comment: Actually, there are three types of BatchData, and two of them are subClass of BatchData. You should test them all. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
