This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/linearfill in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ececdcba39f534073620be6791bf0e44f718ce64 Author: JackieTien97 <[email protected]> AuthorDate: Mon Mar 18 21:19:49 2024 +0800 Fix bug in fill linear --- .../iotdb/db/it/query/IoTDBNullValueFillIT.java | 4 +- .../src/main/codegen/templates/linearFill.ftl | 25 +++--- .../operator/process/fill/linear/LinearFill.java | 49 +++++++++--- .../execution/operator/LinearFillOperatorTest.java | 88 +++++++++++----------- .../tsfile/read/common/block/TsBlockUtil.java | 6 +- .../read/common/block/column/TimeColumn.java | 4 - 6 files changed, 101 insertions(+), 75 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullValueFillIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullValueFillIT.java index cfcb590c26e..36cc6918450 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullValueFillIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBNullValueFillIT.java @@ -47,7 +47,7 @@ public class IoTDBNullValueFillIT { * 3, 3, null, 3.0, null, false, null<br> * 4, null, 4, null, 4.0, null, t4<br> * 5, 5, 5, 5.0, 5.0, false, t5<br> - * 6, null, 6, null, null, false, null<br> + * 6, null, 6, null, 6.0, false, null<br> * 7, null, null, null, null, null, null<br> * 8, 8, 8, 8.0, 8.0, true, t8<br> * 9, 9, null, 9.0, null, true, null @@ -217,7 +217,7 @@ public class IoTDBNullValueFillIT { "3,3,3,3.0,3.0,false,null,", "4,4,4,4.0,4.0,null,t4,", "5,5,5,5.0,5.0,false,t5,", - "6,6,6,6.5,6.0,null,t6,", + "6,6,6,6.0,6.0,null,t6,", "8,8,8,8.0,8.0,true,t8,", "9,9,null,9.0,null,true,null," }; diff --git a/iotdb-core/datanode/src/main/codegen/templates/linearFill.ftl b/iotdb-core/datanode/src/main/codegen/templates/linearFill.ftl index d87467005b0..41ffcc9dd7c 100644 --- a/iotdb-core/datanode/src/main/codegen/templates/linearFill.ftl +++ b/iotdb-core/datanode/src/main/codegen/templates/linearFill.ftl @@ -28,6 +28,7 @@ import org.apache.iotdb.tsfile.read.common.block.column.Column; import org.apache.iotdb.tsfile.read.common.block.column.${type.column}; import org.apache.iotdb.tsfile.read.common.block.column.${type.column}Builder; + import java.util.Optional; /* @@ -49,8 +50,8 @@ public class ${className} extends LinearFill { } @Override - void fillValue(Object array, int index) { - ((${type.dataType}[]) array)[index] = getFilledValue(); + void fillValue(Object array, int index, double factor) { + ((${type.dataType}[]) array)[index] = getFilledValue(factor); } @Override @@ -64,9 +65,13 @@ public class ${className} extends LinearFill { } @Override - Column createFilledValueColumn() { - ${type.dataType} filledValue = getFilledValue(); - return new ${type.column}(1, Optional.empty(), new ${type.dataType}[] {filledValue}); + Column createFilledValueColumn(double[] factors) { + int size = factors.length; + ${type.dataType}[] filledValue = new ${type.dataType}[size]; + for (int i = 0; i < size; i++) { + filledValue[i] = getFilledValue(factors[i]); + } + return new ${type.column}(size, Optional.empty(), filledValue); } @Override @@ -98,14 +103,8 @@ public class ${className} extends LinearFill { this.nextValueInCurrentColumn = this.nextValue; } - private ${type.dataType} getFilledValue() { - <#if type.dataType == "double"> - return (previousValue + nextValueInCurrentColumn) / 2.0; - <#elseif type.dataType == "float"> - return (previousValue + nextValueInCurrentColumn) / 2.0f; - <#else> - return (previousValue + nextValueInCurrentColumn) / 2; - </#if> + private ${type.dataType} getFilledValue(double factor) { + return (${type.dataType}) (previousValue + (nextValueInCurrentColumn - previousValue) * factor); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java index 2b10ec4ca94..0c534e2a11a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/linear/LinearFill.java @@ -42,6 +42,13 @@ public abstract class LinearFill implements ILinearFill { // next row index in current column which is not null private long nextRowIndexInCurrentColumn = -1; + // previous time coresponding to previous not null value + protected long previousTime = -1; + + private long nextTime = -1; + + protected long nextTimeInCurrentColumn = -1; + @Override public Column fill(TimeColumn timeColumn, Column valueColumn, long startRowIndex) { int size = valueColumn.getPositionCount(); @@ -53,6 +60,7 @@ public abstract class LinearFill implements ILinearFill { if (!valueColumn.mayHaveNull()) { previousIsNull = false; // update the value using last non-null value + previousTime = timeColumn.getEndTime(); updatePreviousValue(valueColumn, valueColumn.getPositionCount() - 1); return valueColumn; } @@ -69,11 +77,13 @@ public abstract class LinearFill implements ILinearFill { for (int i = 0; i < size; i++) { // current value is null, we need to fill it if (valueColumn.isNull(i)) { - hasNullValue = fill(startRowIndex, i, isNull, valueColumn, array) || hasNullValue; + hasNullValue = + fill(startRowIndex, i, isNull, timeColumn, valueColumn, array) || hasNullValue; } else { // current is not null // fill value using its own value fillValue(valueColumn, i, array); // update previous value + previousTime = timeColumn.getLong(i); updatePreviousValue(valueColumn, i); previousIsNull = false; } @@ -83,7 +93,7 @@ public abstract class LinearFill implements ILinearFill { } private Column doWithAllNulls( - long startRowIndex, int size, Column timeColumn, Column valueColumn) { + long startRowIndex, int size, TimeColumn timeColumn, Column valueColumn) { // previous value is null or next value is null, we just return NULL_VALUE_BLOCK if (previousIsNull || nextRowIndex < startRowIndex) { return new RunLengthEncodedColumn(createNullValueColumn(), size); @@ -91,26 +101,44 @@ public abstract class LinearFill implements ILinearFill { prepareForNextValueInCurrentColumn( startRowIndex + timeColumn.getPositionCount() - 1, timeColumn.getPositionCount(), + timeColumn, valueColumn); - return new RunLengthEncodedColumn(createFilledValueColumn(), size); + double[] factors = new double[size]; + for (int i = 0; i < size; i++) { + factors[i] = getFactor(timeColumn.getLong(i)); + } + return createFilledValueColumn(factors); } } private boolean fill( - long startRowIndex, int i, boolean[] isNull, Column valueColumn, Object array) { + long startRowIndex, + int i, + boolean[] isNull, + TimeColumn timeColumn, + Column valueColumn, + Object array) { long currentRowIndex = startRowIndex + i; - prepareForNextValueInCurrentColumn(currentRowIndex, i + 1, valueColumn); + prepareForNextValueInCurrentColumn(currentRowIndex, i + 1, timeColumn, valueColumn); // we don't fill it, if either previous value or next value is null if (previousIsNull || nextIsNull(currentRowIndex)) { isNull[i] = true; return true; } else { // fill value using previous and next value - fillValue(array, i); + // factor is (x - x0) / (x1 - x0) + double factor = getFactor(timeColumn.getLong(i)); + fillValue(array, i, factor); return false; } } + private double getFactor(long currentTime) { + return nextTimeInCurrentColumn - previousTime == 0 + ? 0.0 + : ((double) (currentTime - previousTime)) / (nextTimeInCurrentColumn - previousTime); + } + /** * Whether need prepare for next. * @@ -138,6 +166,7 @@ public abstract class LinearFill implements ILinearFill { for (int i = 0; i < nextValueColumn.getPositionCount(); i++) { if (!nextValueColumn.isNull(i)) { updateNextValue(nextValueColumn, i); + this.nextTime = nextTimeColumn.getLong(i); this.nextRowIndex = startRowIndex + i; return true; } @@ -150,13 +179,14 @@ public abstract class LinearFill implements ILinearFill { } private void prepareForNextValueInCurrentColumn( - long currentRowIndex, int startIndex, Column valueColumn) { + long currentRowIndex, int startIndex, TimeColumn timeColumn, Column valueColumn) { if (currentRowIndex <= nextRowIndexInCurrentColumn) { return; } for (int i = startIndex; i < valueColumn.getPositionCount(); i++) { if (!valueColumn.isNull(i)) { this.nextRowIndexInCurrentColumn = currentRowIndex + (i - startIndex + 1); + this.nextTimeInCurrentColumn = timeColumn.getLong(i); updateNextValueInCurrentColumn(valueColumn, i); return; } @@ -164,18 +194,19 @@ public abstract class LinearFill implements ILinearFill { // current column's value is not enough for filling, we should use value of next Column this.nextRowIndexInCurrentColumn = this.nextRowIndex; + this.nextTimeInCurrentColumn = this.nextTime; updateNextValueInCurrentColumn(); } abstract void fillValue(Column column, int index, Object array); - abstract void fillValue(Object array, int index); + abstract void fillValue(Object array, int index, double factor); abstract Object createValueArray(int size); abstract Column createNullValueColumn(); - abstract Column createFilledValueColumn(); + abstract Column createFilledValueColumn(double[] factors1); abstract Column createFilledValueColumn( Object array, boolean[] isNull, boolean hasNullValue, int size); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java index 5e59ac9dba7..dc312880065 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java @@ -190,23 +190,23 @@ public class LinearFillOperatorTest { new float[][][] { { {1.0f, 0.0f, 3.0f, 4.0f}, - {11.0f, 12.0f, 13.0f, 39.0f}, - {21.0f, 22.0f, 28.0f, 39.0f}, - {36.0f, 32.0f, 28.0f, 39.0f}, - {36.0f, 47.0f, 43.0f, 39.0f} + {11.0f, 12.0f, 13.0f, 14.0f}, + {21.0f, 22.0f, 23.0f, 24.0f}, + {31.0f, 32.0f, 33.0f, 34.0f}, + {41.0f, 42.0f, 43.0f, 44.0f} }, { - {51.0f, 47.0f, 53.0f, 39.0f}, - {61.0f, 62.0f, 63.0f, 39.0f}, - {71.0f, 72.0f, 78.0f, 74.0f}, - {86.0f, 82.0f, 78.0f, 94.0f}, - {86.0f, 97.0f, 93.0f, 94.0f} + {51.0f, 52.0f, 53.0f, 54.0f}, + {61.0f, 62.0f, 63.0f, 64.0f}, + {71.0f, 72.0f, 73.0f, 74.0f}, + {81.0f, 82.0f, 83.0f, 84.0f}, + {91.0f, 92.0f, 93.0f, 94.0f} }, { - {101.0f, 97.0f, 103.0f, 94.0f}, + {101.0f, 102.0f, 103.0f, 104.0f}, {111.0f, 112.0f, 113.0f, 114.0f}, - {121.0f, 122.0f, 128.0f, 124.0f}, - {0.0f, 132.0f, 128.0f, 0.0f}, + {121.0f, 122.0f, 123.0f, 124.0f}, + {0.0f, 132.0f, 133.0f, 0.0f}, {0.0f, 0.0f, 143.0f, 0.0f} } }; @@ -407,23 +407,23 @@ public class LinearFillOperatorTest { new float[][][] { { {1.0f, 0.0f, 3.0f, 4.0f}, - {11.0f, 12.0f, 13.0f, 39.0f}, - {21.0f, 22.0f, 28.0f, 39.0f}, - {36.0f, 32.0f, 28.0f, 39.0f}, - {36.0f, 47.0f, 43.0f, 39.0f} + {11.0f, 12.0f, 13.0f, 14.0f}, + {21.0f, 22.0f, 23.0f, 24.0f}, + {31.0f, 32.0f, 33.0f, 34.0f}, + {41.0f, 42.0f, 43.0f, 44.0f} }, { - {51.0f, 47.0f, 53.0f, 39.0f}, - {61.0f, 62.0f, 63.0f, 39.0f}, - {71.0f, 72.0f, 78.0f, 74.0f}, - {86.0f, 82.0f, 78.0f, 94.0f}, - {86.0f, 97.0f, 93.0f, 94.0f} + {51.0f, 52.0f, 53.0f, 54.0f}, + {61.0f, 62.0f, 63.0f, 64.0f}, + {71.0f, 72.0f, 73.0f, 74.0f}, + {81.0f, 82.0f, 83.0f, 84.0f}, + {91.0f, 92.0f, 93.0f, 94.0f} }, { - {101.0f, 97.0f, 103.0f, 94.0f}, + {101.0f, 102.0f, 103.0f, 104.0f}, {111.0f, 112.0f, 113.0f, 114.0f}, - {121.0f, 122.0f, 128.0f, 124.0f}, - {0.0f, 132.0f, 128.0f, 0.0f}, + {121.0f, 122.0f, 123.0f, 124.0f}, + {0.0f, 132.0f, 133.0f, 0.0f}, {0.0f, 0.0f, 143.0f, 0.0f} } }; @@ -626,20 +626,20 @@ public class LinearFillOperatorTest { new float[][][] { { {1.0f, 0.0f, 3.0f, 4.0f}, - {11.0f, 12.0f, 13.0f, 39.0f}, - {21.0f, 22.0f, 58.0f, 39.0f}, - {36.0f, 32.0f, 58.0f, 39.0f}, - {36.0f, 47.0f, 58.0f, 39.0f} + {11.0f, 12.0f, 13.0f, 14.0f}, + {21.0f, 22.0f, 23.0f, 24.0f}, + {31.0f, 32.0f, 33.0f, 34.0f}, + {41.0f, 42.0f, 43.0f, 44.0f} }, { - {51.0f, 47.0f, 58.0f, 39.0f}, - {61.0f, 62.0f, 58.0f, 39.0f}, - {71.0f, 72.0f, 58.0f, 74.0f}, - {86.0f, 82.0f, 58.0f, 94.0f}, - {86.0f, 97.0f, 58.0f, 94.0f} + {51.0f, 52.0f, 53.0f, 54.0f}, + {61.0f, 62.0f, 63.0f, 64.0f}, + {71.0f, 72.0f, 73.0f, 74.0f}, + {81.0f, 82.0f, 83.0f, 84.0f}, + {91.0f, 92.0f, 93.0f, 94.0f} }, { - {101.0f, 97.0f, 103.0f, 94.0f}, + {101.0f, 102.0f, 103.0f, 104.0f}, {111.0f, 112.0f, 0.0f, 114.0f}, {121.0f, 122.0f, 0.0f, 124.0f}, {0.0f, 132.0f, 0.0f, 0.0f}, @@ -844,20 +844,20 @@ public class LinearFillOperatorTest { new float[][][] { { {1.0f, 0.0f, 3.0f, 4.0f}, - {11.0f, 12.0f, 13.0f, 39.0f}, - {21.0f, 22.0f, 58.0f, 39.0f}, - {36.0f, 32.0f, 58.0f, 39.0f}, - {36.0f, 47.0f, 58.0f, 39.0f} + {11.0f, 12.0f, 13.0f, 14.0f}, + {21.0f, 22.0f, 23.0f, 24.0f}, + {31.0f, 32.0f, 33.0f, 34.0f}, + {41.0f, 42.0f, 43.0f, 44.0f} }, { - {51.0f, 47.0f, 58.0f, 39.0f}, - {61.0f, 62.0f, 58.0f, 39.0f}, - {71.0f, 72.0f, 58.0f, 74.0f}, - {86.0f, 82.0f, 58.0f, 94.0f}, - {86.0f, 97.0f, 58.0f, 94.0f} + {51.0f, 52.0f, 53.0f, 54.0f}, + {61.0f, 62.0f, 63.0f, 64.0f}, + {71.0f, 72.0f, 73.0f, 74.0f}, + {81.0f, 82.0f, 83.0f, 84.0f}, + {91.0f, 92.0f, 93.0f, 94.0f} }, { - {101.0f, 97.0f, 103.0f, 94.0f}, + {101.0f, 102.0f, 103.0f, 104.0f}, {111.0f, 112.0f, 0.0f, 114.0f}, {121.0f, 122.0f, 0.0f, 124.0f}, {0.0f, 132.0f, 0.0f, 0.0f}, diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockUtil.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockUtil.java index 3bf472088a7..2206954fa12 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockUtil.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockUtil.java @@ -49,19 +49,19 @@ public class TsBlockUtil { while (left < right) { mid = (left + right) >> 1; - if (timeColumn.getLongWithoutCheck(mid) < targetTime) { + if (timeColumn.getLong(mid) < targetTime) { if (ascending) { left = mid + 1; } else { right = mid; } - } else if (timeColumn.getLongWithoutCheck(mid) > targetTime) { + } else if (timeColumn.getLong(mid) > targetTime) { if (ascending) { right = mid; } else { left = mid + 1; } - } else if (timeColumn.getLongWithoutCheck(mid) == targetTime) { + } else if (timeColumn.getLong(mid) == targetTime) { return mid; } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java index a18d5fcf3dd..88c9c3605ee 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java @@ -75,10 +75,6 @@ public class TimeColumn implements Column { return values[position + arrayOffset]; } - public long getLongWithoutCheck(int position) { - return values[position + arrayOffset]; - } - @Override public Object getObject(int position) { return getLong(position);
