This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch research/encoding-reorder in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 400237ab4f844ab8c3c004dfd6be7f7a53b82aa6 Author: xjz17 <[email protected]> AuthorDate: Sun Dec 3 16:34:52 2023 +0800 update --- .../iotdb/tsfile/encoding/REGERDoubleTest.java | 39 ++++++---- .../tsfile/encoding/REGERFloatDecreaseTest.java | 8 +- .../iotdb/tsfile/encoding/REGERFloatTest.java | 86 ++++++++-------------- .../iotdb/tsfile/encoding/RegerPFloatTest.java | 45 ++++++----- 4 files changed, 86 insertions(+), 92 deletions(-) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java index b5c4f78c435..4342bfee274 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java @@ -2507,7 +2507,7 @@ public class REGERDoubleTest { // System.out.println((data[i * block_size+1])); // System.out.println(getTime(data[i * block_size+1])); for (int j = 0; j < block_size; j++) { - long tmp_j = data[j + i * block_size];// - min_time; + long tmp_j = data[j + i * block_size] - min_time; // System.out.println(getTime(data[j + i * block_size])); // System.out.println(getTime(data[i * block_size])); ts_block[j] = tmp_j; @@ -2558,7 +2558,7 @@ public class REGERDoubleTest { // int min_value = Integer.MAX_VALUE; for (int j = 0; j < end; j++) { - long tmp_j = data[j + i * block_size];// - min_time; + long tmp_j = data[j + i * block_size] - min_time; ts_block[j] = tmp_j; ts_block_value[j] = combine2Int(getValue(tmp_j),getTime(tmp_j)); @@ -2593,7 +2593,8 @@ public class REGERDoubleTest { } - + int2Bytes(getTime(min_time),encode_pos,cur_byte); + encode_pos += 4; int[] reorder_length = new int[5]; double[] theta_reorder = new double[4]; int[] time_length = new int[5];// length,max_bit_width_interval,max_bit_width_value,max_bit_width_deviation @@ -2608,30 +2609,35 @@ public class REGERDoubleTest { int pos_ts_block_partition = 0; - if(third_value.length>0){ - for (long datum : ts_block) { - if (getValue(datum) > third_value[third_value.length - 1]) { + if (third_value.length > 0) { + for(int j=block_size-1;j>=0;j--){ + long datum = ts_block[j]; + if (getValue(datum) <= third_value[0]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } - for (int third_i = third_value.length - 1; third_i > 0; third_i--) { - for (long datum : ts_block) { + for (int third_i = 1; third_i <= third_value.length - 1; third_i++) { + for (int j = block_size - 1; j >= 0; j--) { + long datum = ts_block[j]; if (getValue(datum) <= third_value[third_i] && getValue(datum) > third_value[third_i - 1]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } } - for (long datum : ts_block) { - if (getValue(datum) <= third_value[0]) { + for (int j = block_size - 1; j >= 0; j--) { + long datum = ts_block[j]; + if (getValue(datum) > third_value[third_value.length - 1]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } + } + trainParameter(ts_block_partition,block_size,theta_partition); long[] ts_block_delta_partition = getEncodeBitsRegressionNoTrain(ts_block_partition, block_size, partition_length, theta_partition,segment_size); @@ -2767,9 +2773,12 @@ public class REGERDoubleTest { public static int REGERBlockDecoder(byte[] encoded, int decode_pos, int[][] value_list, int block_size, int segment_size, int[] value_pos_arr) { + int min_time_0 = bytes2Integer(encoded, decode_pos, 4); + decode_pos += 4; + int time0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; - value_list[value_pos_arr[0]][0] = time0; + value_list[value_pos_arr[0]][0] = time0+min_time_0; int value0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; value_list[value_pos_arr[0]][1] = value0; @@ -2842,7 +2851,7 @@ public class REGERDoubleTest { for(;pos_decode_time_result<length_decode_time_result;pos_decode_time_result++){ pre_time = (int) (theta_time0 + theta_time1 * (float)pre_time ) + decode_time_result[pos_decode_time_result] + min_time; pre_value = (int) (theta_value0 + theta_value1 * (float)pre_value ) + decode_value_result[pos_decode_time_result] + min_value; - value_list[value_pos_arr[0]][0] = pre_time; + value_list[value_pos_arr[0]][0] = pre_time + min_time_0; value_list[value_pos_arr[0]][1] = pre_value; value_pos_arr[0] ++; } @@ -2854,12 +2863,14 @@ public class REGERDoubleTest { public static int REGERBlockDecoderValue(byte[] encoded, int decode_pos, int[][] value_list, int block_size, int segment_size, int[] value_pos_arr) { + int min_time_0 = bytes2Integer(encoded, decode_pos, 4); + decode_pos += 4; int time0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; value_list[value_pos_arr[0]][1] = time0; int value0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; - value_list[value_pos_arr[0]][0] = value0; + value_list[value_pos_arr[0]][0] = value0+min_time_0; value_pos_arr[0]++; @@ -2930,7 +2941,7 @@ public class REGERDoubleTest { pre_time = (int) (theta_time0 + theta_time1 * (float)pre_time ) + decode_time_result[pos_decode_time_result] + min_time; pre_value = (int) (theta_value0 + theta_value1 * (float)pre_value ) + decode_value_result[pos_decode_time_result] + min_value; value_list[value_pos_arr[0]][1] = pre_time; - value_list[value_pos_arr[0]][0] = pre_value; + value_list[value_pos_arr[0]][0] = pre_value+min_time_0; value_pos_arr[0] ++; } } diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java index 1e17fb891cc..3cc0bee6ee6 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java @@ -2525,7 +2525,7 @@ public class REGERFloatDecreaseTest { // System.out.println((data[i * block_size+1])); // System.out.println(getTime(data[i * block_size+1])); for (int j = 0; j < block_size; j++) { - long tmp_j = data[j + i * block_size]- min_time; + long tmp_j = data[j + i * block_size];//- min_time; // System.out.println(getTime(data[j + i * block_size])); // System.out.println(getTime(data[i * block_size])); ts_block[j] = tmp_j; @@ -2576,7 +2576,7 @@ public class REGERFloatDecreaseTest { // int min_value = Integer.MAX_VALUE; for (int j = 0; j < end; j++) { - long tmp_j = data[j + i * block_size] - min_time; + long tmp_j = data[j + i * block_size];// - min_time; ts_block[j] = tmp_j; ts_block_value[j] = combine2Int(getValue(tmp_j), getTime(tmp_j)); @@ -2609,8 +2609,8 @@ public class REGERFloatDecreaseTest { // third_value= findMinIndex(kernelDensity); block_size = supply_length; } - int2Bytes(getTime(min_time),encode_pos,cur_byte); - encode_pos += 4; +// int2Bytes(getTime(min_time),encode_pos,cur_byte); +// encode_pos += 4; // printTSBlock(ts_block); // printTSBlock(ts_block_value); diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java index fbd52cf1034..c25edec3cc6 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java @@ -2661,7 +2661,7 @@ public class REGERFloatTest { private static int REGERBlockEncoder(long[] data, int i, int block_size, int supply_length, int[] third_value, int segment_size, int encode_pos, byte[] cur_byte, int[] block_sort) { -// long min_time = (long) getTime(data[i * block_size]) << 32; + long min_time = (long) getTime(data[i * block_size]) << 32; long[] ts_block; long[] ts_block_value; long[] ts_block_partition; @@ -2676,7 +2676,7 @@ public class REGERFloatTest { // System.out.println((data[i * block_size+1])); // System.out.println(getTime(data[i * block_size+1])); for (int j = 0; j < block_size; j++) { - long tmp_j = data[j + i * block_size];//- min_time; + long tmp_j = data[j + i * block_size] - min_time; // System.out.println(getTime(data[j + i * block_size])); // System.out.println(getTime(data[i * block_size])); ts_block[j] = tmp_j; @@ -2727,7 +2727,7 @@ public class REGERFloatTest { // int min_value = Integer.MAX_VALUE; for (int j = 0; j < end; j++) { - long tmp_j = data[j + i * block_size];// - min_time; + long tmp_j = data[j + i * block_size] - min_time; ts_block[j] = tmp_j; ts_block_value[j] = combine2Int(getValue(tmp_j), getTime(tmp_j)); @@ -2761,8 +2761,6 @@ public class REGERFloatTest { block_size = supply_length; } -// printTSBlock(ts_block); -// printTSBlock(ts_block_value); int[] reorder_length = new int[5]; float[] theta_reorder = new float[4]; @@ -2778,26 +2776,30 @@ public class REGERFloatTest { int pos_ts_block_partition = 0; if (third_value.length > 0) { - for (long datum : ts_block) { - if (getValue(datum) > third_value[third_value.length - 1]) { + for(int j=block_size-1;j>=0;j--){ + long datum = ts_block[j]; + if (getValue(datum) <= third_value[0]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } - for (int third_i = third_value.length - 1; third_i > 0; third_i--) { - for (long datum : ts_block) { + for (int third_i = 1; third_i <= third_value.length - 1; third_i++) { + for (int j = block_size - 1; j >= 0; j--) { + long datum = ts_block[j]; if (getValue(datum) <= third_value[third_i] && getValue(datum) > third_value[third_i - 1]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } } - for (long datum : ts_block) { - if (getValue(datum) <= third_value[0]) { + for (int j = block_size - 1; j >= 0; j--) { + long datum = ts_block[j]; + if (getValue(datum) > third_value[third_value.length - 1]) { ts_block_partition[pos_ts_block_partition] = datum; pos_ts_block_partition++; } } + } @@ -2809,41 +2811,32 @@ public class REGERFloatTest { long[] ts_block_delta_reorder = getEncodeBitsRegressionNoTrain(ts_block_value, block_size, reorder_length, theta_reorder, segment_size); -// ReorderingTimeSeries(ts_block_value, reorder_length, theta_reorder, segment_size); + ReorderingTimeSeries(ts_block_value, reorder_length, theta_reorder, segment_size); int segment_n = (block_size - 1) / segment_size; long[] bit_width_segments = new long[segment_n]; - + int2Bytes(getTime(min_time),encode_pos,cur_byte); + encode_pos += 4; int choose = min3(time_length[0], partition_length[0], reorder_length[0]); if (choose == 0) { -// System.out.println("time"); -// System.out.println(Arrays.toString(time_length)); -// intByte2Bytes(0, encode_pos, cur_byte); -// encode_pos ++; + block_sort[i] = 0; ts_block_delta_time = ReorderingTimeSeries(ts_block, time_length, theta_time, segment_size); bit_width_segments = segmentBitPacking(ts_block_delta_time, block_size, segment_size); -// ts_block_delta_time[0] += min_time; + + encode_pos = encodeSegment2Bytes(ts_block_delta_time, bit_width_segments, time_length, segment_size, theta_time, encode_pos, cur_byte); // System.out.println(Arrays.toString(time_length)); } else if (choose == 1) { -// System.out.println("partition"); -// System.out.println(Arrays.toString(partition_length)); -// intByte2Bytes(0, encode_pos, cur_byte); -// encode_pos ++; + block_sort[i] = 0; ts_block_delta_partition = ReorderingTimeSeries(ts_block_partition, partition_length, theta_partition, segment_size); bit_width_segments = segmentBitPacking(ts_block_delta_partition, block_size, segment_size); -// ts_block_delta_partition[0] += min_time; + encode_pos = encodeSegment2Bytes(ts_block_delta_partition, bit_width_segments, partition_length, segment_size, theta_partition, encode_pos, cur_byte); -// System.out.println(Arrays.toString(partition_length)); } else if (choose == 2) { -// System.out.println("value"); -// System.out.println(Arrays.toString(reorder_length)); -// intByte2Bytes(1, encode_pos, cur_byte); -// encode_pos ++; block_sort[i] = 1; ts_block_delta_reorder = ReorderingTimeSeries(ts_block_value, reorder_length, theta_reorder, segment_size); @@ -2852,27 +2845,8 @@ public class REGERFloatTest { } -// int segment_n = (block_size - 1) / segment_size; -// long[] bit_width_segments = new long[segment_n]; -// -// if(time_length[0] > partition_length[0]){ -// -// ts_block_delta_partition = ReorderingTimeSeries(ts_block_partition, partition_length, theta_partition, segment_size); -// -// bit_width_segments = segmentBitPacking(ts_block_delta_partition, block_size, segment_size); -// encode_pos = encodeSegment2Bytes(ts_block_delta_partition, bit_width_segments, partition_length, segment_size, theta_partition, encode_pos, cur_byte); -// -// }else{ -// ts_block_delta_time = ReorderingTimeSeries(ts_block, time_length, theta_time, segment_size); -// -// bit_width_segments = segmentBitPacking(ts_block_delta_time, block_size, segment_size); -// encode_pos = encodeSegment2Bytes(ts_block_delta_time, bit_width_segments, time_length, segment_size, theta_time, encode_pos, cur_byte); -// -// } - -// System.out.println(encode_pos); - return encode_pos; + } public static int ReorderingRegressionEncoder(long[] data, int block_size, int[] third_value, int segment_size, byte[] encoded_result) { @@ -2943,11 +2917,12 @@ public class REGERFloatTest { public static int REGERBlockDecoder(byte[] encoded, int decode_pos, int[][] value_list, int block_size, int segment_size, int[] value_pos_arr) { - + int min_time_0 = bytes2Integer(encoded, decode_pos, 4); + decode_pos += 4; int time0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; - value_list[value_pos_arr[0]][0] = time0; + value_list[value_pos_arr[0]][0] = time0+min_time_0; int value0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; value_list[value_pos_arr[0]][1] = value0; @@ -3021,7 +2996,7 @@ public class REGERFloatTest { for(;pos_decode_time_result<length_decode_time_result;pos_decode_time_result++){ pre_time = (int) (theta_time0 + theta_time1 * (float)pre_time ) + decode_time_result[pos_decode_time_result] + min_time; pre_value = (int) (theta_value0 + theta_value1 * (float)pre_value ) + decode_value_result[pos_decode_time_result] + min_value; - value_list[value_pos_arr[0]][0] = pre_time; + value_list[value_pos_arr[0]][0] = pre_time+min_time_0; value_list[value_pos_arr[0]][1] = pre_value; value_pos_arr[0] ++; } @@ -3032,14 +3007,15 @@ public class REGERFloatTest { public static int REGERBlockDecoderValue(byte[] encoded, int decode_pos, int[][] value_list, int block_size, int segment_size, int[] value_pos_arr) { - + int min_time_0 = bytes2Integer(encoded, decode_pos, 4); + decode_pos += 4; int time0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; - value_list[value_pos_arr[0]][1] = time0; + value_list[value_pos_arr[0]][1] = time0 ; int value0 = bytes2Integer(encoded, decode_pos, 4); decode_pos += 4; - value_list[value_pos_arr[0]][0] = value0; + value_list[value_pos_arr[0]][0] = value0+min_time_0; @@ -3113,7 +3089,7 @@ public class REGERFloatTest { pre_time = (int) (theta_time0 + theta_time1 * (float)pre_time ) + decode_time_result[pos_decode_time_result] + min_time; pre_value = (int) (theta_value0 + theta_value1 * (float)pre_value ) + decode_value_result[pos_decode_time_result] + min_value; value_list[value_pos_arr[0]][1] = pre_time; - value_list[value_pos_arr[0]][0] = pre_value; + value_list[value_pos_arr[0]][0] = pre_value+min_time_0; value_pos_arr[0] ++; } } diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java index bae409913f0..14edb0a50b2 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java @@ -1680,7 +1680,8 @@ public class RegerPFloatTest { } block_size = supply_length; } - + int2Bytes(min_time,encode_pos,cur_byte); + encode_pos += 4; // System.out.println(Arrays.deepToString(data)); // System.out.println(Arrays.deepToString(ts_block)); @@ -1716,15 +1717,19 @@ public class RegerPFloatTest { // System.out.println((Arrays.toString(time_length))); int pos_ts_block_partition = 0; - for (int[] datum : ts_block) { - if (datum[1] > third_value[third_value.length - 1]) { - ts_block_partition[pos_ts_block_partition][0] = datum[0]; - ts_block_partition[pos_ts_block_partition][1] = datum[1]; - pos_ts_block_partition++; + if (third_value.length > 0) { + + for(int j=block_size-1;j>=0;j--){ + int[] datum = ts_block[j]; + if (datum[1] <= third_value[0]) { + ts_block_partition[pos_ts_block_partition][0] = datum[0]; + ts_block_partition[pos_ts_block_partition][1] = datum[1]; + pos_ts_block_partition++; + } } - } - for (int third_i = third_value.length - 1; third_i > 0; third_i--) { - for (int[] datum : ts_block) { + for (int third_i = 1; third_i <third_value.length ; third_i++) { + for(int j=block_size-1;j>=0;j--){ + int[] datum = ts_block[j]; if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) { ts_block_partition[pos_ts_block_partition][0] = datum[0]; ts_block_partition[pos_ts_block_partition][1] = datum[1]; @@ -1732,11 +1737,13 @@ public class RegerPFloatTest { } } } - for (int[] datum : ts_block) { - if (datum[1] <= third_value[0]) { - ts_block_partition[pos_ts_block_partition][0] = datum[0]; - ts_block_partition[pos_ts_block_partition][1] = datum[1]; - pos_ts_block_partition++; + for(int j=block_size-1;j>=0;j--){ + int[] datum = ts_block[j]; + if (datum[1] > third_value[third_value.length - 1]) { + ts_block_partition[pos_ts_block_partition][0] = datum[0]; + ts_block_partition[pos_ts_block_partition][1] = datum[1]; + pos_ts_block_partition++; + } } } terminate(ts_block_partition, theta, p); @@ -2480,7 +2487,7 @@ public class RegerPFloatTest { for (int i = 0; i < dataset_name.size(); i++) { input_path_list.add(input_parent_dir + dataset_name.get(i)); - dataset_block_size.add(1024); + dataset_block_size.add(512); } output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0 @@ -2513,9 +2520,9 @@ public class RegerPFloatTest { // 0 2 6 7 - int[] file_lists = {1,9}; - for (int file_i : file_lists) { -// for (int file_i = 0; file_i < input_path_list.size(); file_i++) { +// int[] file_lists = {1,9}; +// for (int file_i : file_lists) { + for (int file_i = 0; file_i < input_path_list.size(); file_i++) { // for (int file_i = 12; file_i < 14; file_i++) { // String inputPath = input_path_list.get(file_i); @@ -2593,7 +2600,7 @@ public class RegerPFloatTest { data2_arr, dataset_block_size.get(file_i), dataset_third.get(file_i), - 8, + 16, p, 1, encoded_result);
