This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch clone_value_array_when_insert_nonaligned_tablet_with_null in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c89987e29f2e3c788c878bc2ffc70b336052dbe2 Author: HTHou <[email protected]> AuthorDate: Tue Jan 7 15:16:08 2025 +0800 Clone value array when insert non aligned tablet with null --- .../iotdb/db/utils/datastructure/BinaryTVList.java | 6 +++++- .../db/utils/datastructure/BooleanTVList.java | 6 +++++- .../iotdb/db/utils/datastructure/DoubleTVList.java | 6 +++++- .../iotdb/db/utils/datastructure/FloatTVList.java | 6 +++++- .../iotdb/db/utils/datastructure/IntTVList.java | 6 +++++- .../iotdb/db/utils/datastructure/LongTVList.java | 6 +++++- .../storageengine/dataregion/DataRegionTest.java | 24 ++++++++++++++++++---- 7 files changed, 50 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java index bb8cd6e513c..b92be71a9d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java @@ -210,11 +210,15 @@ public abstract class BinaryTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + Binary[] clonedValue = new Binary[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java index 00b0c3de8fd..74d5a8f8a0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java @@ -169,11 +169,15 @@ public abstract class BooleanTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + boolean[] clonedValue = new boolean[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java index b7cc3336d10..f6445c7b7ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java @@ -174,11 +174,15 @@ public abstract class DoubleTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + double[] clonedValue = new double[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index 43a208ecccf..62d307cc142 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -174,11 +174,15 @@ public abstract class FloatTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + float[] clonedValue = new float[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index 162f1fd1112..48dd833afb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -167,11 +167,15 @@ public abstract class IntTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + int[] clonedValue = new int[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java index 4c8f4ccea9e..221987f9449 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java @@ -167,11 +167,15 @@ public abstract class LongTVList extends TVList { // constraint: time.length + timeIdxOffset == value.length int timeIdxOffset = 0; if (bitMap != null && !bitMap.isAllUnmarked()) { - // time array is a reference, should clone necessary time values + // time array is a reference, should clone necessary time array long[] clonedTime = new long[end - start]; System.arraycopy(time, start, clonedTime, 0, end - start); time = clonedTime; timeIdxOffset = start; + // value array is a reference, should clone necessary value array + long[] clonedValue = new long[end - start]; + System.arraycopy(value, start, clonedValue, 0, end - start); + value = clonedValue; // drop null at the end of value array int nullCnt = dropNullValThenUpdateMaxTimeAndSorted(time, value, bitMap, start, end, timeIdxOffset); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index ddce03e6833..1f0ce1e77b0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -90,6 +90,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -385,8 +386,19 @@ public class DataRegionTest { for (int r = 0; r < 100; r++) { times[r] = r; - ((int[]) columns[0])[r] = 1; - ((long[]) columns[1])[r] = 1; + ((int[]) columns[0])[r] = r; + ((long[]) columns[1])[r] = r; + } + + BitMap[] bitMaps = new BitMap[2]; + bitMaps[0] = new BitMap(100); + bitMaps[1] = new BitMap(100); + for (int r = 0; r < 100; r++) { + if (r % 2 == 0) { + bitMaps[0].mark(r); + } else { + bitMaps[1].mark(r); + } } InsertTabletNode insertTabletNode1 = @@ -398,11 +410,15 @@ public class DataRegionTest { dataTypes, measurementSchemas, times, - null, + bitMaps, columns, times.length); - + int hashCode1 = Arrays.hashCode((int[]) columns[0]); + int hashCode2 = Arrays.hashCode((long[]) columns[1]); dataRegion.insertTablet(insertTabletNode1); + // the hashCode should not be changed when insert + Assert.assertEquals(hashCode1, Arrays.hashCode((int[]) columns[0])); + Assert.assertEquals(hashCode2, Arrays.hashCode((long[]) columns[1])); dataRegion.syncCloseAllWorkingTsFileProcessors(); for (int r = 50; r < 149; r++) {
