This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch IOTDB-6243 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit de5bca6445ec2cde349d2e51f97bf4ebf0f38ed5 Author: JackieTien97 <[email protected]> AuthorDate: Thu Nov 9 21:02:43 2023 +0800 try fix --- ...r.java => AbstractMonthIntervalFillFilter.java} | 19 ++++++-- .../fill/filter/FixedIntervalFillFilter.java | 3 +- ...lFilter.java => MonthIntervalMSFillFilter.java} | 19 ++++---- ...lFilter.java => MonthIntervalNSFillFilter.java} | 18 ++++---- ...lFilter.java => MonthIntervalUSFillFilter.java} | 19 ++++---- .../queryengine/plan/analyze/AnalyzeVisitor.java | 5 +- .../db/queryengine/plan/parser/ASTVisitor.java | 6 ++- .../plan/planner/OperatorTreeGenerator.java | 54 ++++++++++++++++++++-- .../plan/planner/plan/node/PlanGraphPrinter.java | 4 +- .../planner/plan/parameter/FillDescriptor.java | 3 +- .../plan/optimization/TestPlanBuilder.java | 6 ++- .../apache/iotdb/tsfile/utils/TimeDuration.java | 8 ++-- 12 files changed, 115 insertions(+), 49 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java similarity index 71% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilter.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java index 1f3c193833b..78b48d2fd89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java @@ -21,23 +21,34 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter; -public class MonthIntervalFillFilter implements IFillFilter { +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; + +public abstract class AbstractMonthIntervalFillFilter implements IFillFilter { // month part of time duration public final int monthDuration; // non-month part of time duration, its precision is same as current time_precision public final long nonMonthDuration; - public MonthIntervalFillFilter(int monthDuration, long nonMonthDuration) { + public final ZoneId zone; + + public AbstractMonthIntervalFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) { this.monthDuration = monthDuration; this.nonMonthDuration = nonMonthDuration; + this.zone = zone; } @Override public boolean needFill(long time, long previousTime) { + long smaller = Math.min(time, previousTime); long greater = Math.max(time, previousTime); - - return false; + Instant instant = Instant.ofEpochSecond(smaller, smaller); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone); + return needFill(localDateTime, greater); } + + abstract boolean needFill(LocalDateTime smaller, long greater); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java index 7b1fdd09550..840148438a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java @@ -32,7 +32,8 @@ public class FixedIntervalFillFilter implements IFillFilter { @Override public boolean needFill(long time, long previousTime) { - // the reason that we use Math.abs is that we may use order by time desc which will cause previousTime is larger than time + // the reason that we use Math.abs is that we may use order by time desc which will cause + // previousTime is larger than time return Math.abs(time - previousTime) <= timeInterval; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java similarity index 60% copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java index 7b1fdd09550..4885152d1c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java @@ -19,20 +19,19 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; -import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter; +import java.time.LocalDateTime; +import java.time.ZoneId; -public class FixedIntervalFillFilter implements IFillFilter { +public class MonthIntervalMSFillFilter extends AbstractMonthIntervalFillFilter { - // the time precision of this field is same as the system time_precision configuration. - private final long timeInterval; - - public FixedIntervalFillFilter(long timeInterval) { - this.timeInterval = timeInterval; + public MonthIntervalMSFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) { + super(monthDuration, nonMonthDuration, zone); } @Override - public boolean needFill(long time, long previousTime) { - // the reason that we use Math.abs is that we may use order by time desc which will cause previousTime is larger than time - return Math.abs(time - previousTime) <= timeInterval; + boolean needFill(LocalDateTime smaller, long greater) { + LocalDateTime localDateTime = + smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000_000L); + return greater >= (localDateTime.getSecond() * 1_000L + localDateTime.getNano() / 1_000_000); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java similarity index 60% copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java index 7b1fdd09550..ec376051dae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java @@ -19,20 +19,18 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; -import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter; +import java.time.LocalDateTime; +import java.time.ZoneId; -public class FixedIntervalFillFilter implements IFillFilter { +public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter { - // the time precision of this field is same as the system time_precision configuration. - private final long timeInterval; - - public FixedIntervalFillFilter(long timeInterval) { - this.timeInterval = timeInterval; + public MonthIntervalNSFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) { + super(monthDuration, nonMonthDuration, zone); } @Override - public boolean needFill(long time, long previousTime) { - // the reason that we use Math.abs is that we may use order by time desc which will cause previousTime is larger than time - return Math.abs(time - previousTime) <= timeInterval; + boolean needFill(LocalDateTime smaller, long greater) { + LocalDateTime localDateTime = smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration); + return greater >= (localDateTime.getSecond() * 1_000_000_000L + localDateTime.getNano()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java similarity index 60% copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java index 7b1fdd09550..68afcda4ead 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java @@ -19,20 +19,19 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; -import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter; +import java.time.LocalDateTime; +import java.time.ZoneId; -public class FixedIntervalFillFilter implements IFillFilter { +public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter { - // the time precision of this field is same as the system time_precision configuration. - private final long timeInterval; - - public FixedIntervalFillFilter(long timeInterval) { - this.timeInterval = timeInterval; + public MonthIntervalUSFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) { + super(monthDuration, nonMonthDuration, zone); } @Override - public boolean needFill(long time, long previousTime) { - // the reason that we use Math.abs is that we may use order by time desc which will cause previousTime is larger than time - return Math.abs(time - previousTime) <= timeInterval; + boolean needFill(LocalDateTime smaller, long greater) { + LocalDateTime localDateTime = + smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000L); + return greater >= (localDateTime.getSecond() * 1_000_000L + localDateTime.getNano() / 1_000); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 3b18c01ce0f..1da5b2368fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -1782,7 +1782,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> FillComponent fillComponent = queryStatement.getFillComponent(); analysis.setFillDescriptor( - new FillDescriptor(fillComponent.getFillPolicy(), fillComponent.getFillValue(), fillComponent.getTimeDurationThreshold())); + new FillDescriptor( + fillComponent.getFillPolicy(), + fillComponent.getFillValue(), + fillComponent.getTimeDurationThreshold())); } private void analyzeDataPartition( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 93cef11658d..e9da29d1df4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1677,9 +1677,11 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { } if (ctx.interval != null) { if (fillComponent.getFillPolicy() != FillPolicy.PREVIOUS) { - throw new SemanticException("Only FILL(PREVIOUS) support specifying the time duration threshold."); + throw new SemanticException( + "Only FILL(PREVIOUS) support specifying the time duration threshold."); } - fillComponent.setTimeDurationThreshold(DateTimeUtils.constructTimeDuration(ctx.interval.getText())); + fillComponent.setTimeDurationThreshold( + DateTimeUtils.constructTimeDuration(ctx.interval.getText())); } return fillComponent; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 26a11276993..4fb7847c00e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -73,6 +73,9 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.fill.constant. import org.apache.iotdb.db.queryengine.execution.operator.process.fill.constant.IntConstantFill; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.constant.LongConstantFill; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.FixedIntervalFillFilter; +import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.MonthIntervalMSFillFilter; +import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.MonthIntervalNSFillFilter; +import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.MonthIntervalUSFillFilter; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.identity.IdentityFill; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.identity.IdentityLinearFill; import org.apache.iotdb.db.queryengine.execution.operator.process.fill.linear.DoubleLinearFill; @@ -224,14 +227,15 @@ import org.apache.iotdb.tsfile.read.filter.operator.Gt; import org.apache.iotdb.tsfile.read.filter.operator.GtEq; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.utils.TimeDuration; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; -import org.apache.iotdb.tsfile.utils.TimeDuration; import java.io.File; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -252,6 +256,7 @@ import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator; import static org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand.TIMESTAMP_EXPRESSION_STRING; import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions.updateFilterUsingTTL; +import static org.apache.iotdb.db.utils.TimestampPrecisionUtils.TIMESTAMP_PRECISION; /** This Visitor is responsible for transferring PlanNode Tree to Operator Tree. */ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionPlanContext> { @@ -937,7 +942,18 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP case PREVIOUS: context.getTimeSliceAllocator().recordExecutionWeight(operatorContext, 1); return new FillOperator( - operatorContext, getPreviousFill(inputColumns, inputDataTypes, descriptor.getTimeDurationThreshold()), child); + operatorContext, + getPreviousFill( + inputColumns, + inputDataTypes, + descriptor.getTimeDurationThreshold(), + ZoneId.of( + context + .getDriverContext() + .getFragmentInstanceContext() + .getSessionInfo() + .getZoneId())), + child); case LINEAR: context.getTimeSliceAllocator().recordExecutionWeight(operatorContext, 1); return new LinearFillOperator( @@ -981,14 +997,44 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP return constantFill; } - private IFill[] getPreviousFill(int inputColumns, List<TSDataType> inputDataTypes, TimeDuration timeDurationThreshold) { + private IFill[] getPreviousFill( + int inputColumns, + List<TSDataType> inputDataTypes, + TimeDuration timeDurationThreshold, + ZoneId zoneId) { IFillFilter filter; if (timeDurationThreshold == null) { filter = IFillFilter.TRUE; } else if (!timeDurationThreshold.containsMonth()) { filter = new FixedIntervalFillFilter(timeDurationThreshold.nonMonthDuration); } else { - + switch (TIMESTAMP_PRECISION) { + case "ms": + filter = + new MonthIntervalMSFillFilter( + timeDurationThreshold.monthDuration, + timeDurationThreshold.nonMonthDuration, + zoneId); + break; + case "us": + filter = + new MonthIntervalUSFillFilter( + timeDurationThreshold.monthDuration, + timeDurationThreshold.nonMonthDuration, + zoneId); + break; + case "ns": + filter = + new MonthIntervalNSFillFilter( + timeDurationThreshold.monthDuration, + timeDurationThreshold.nonMonthDuration, + zoneId); + break; + default: + // this case will never reach + throw new UnsupportedOperationException( + "not supported time_precision: " + TIMESTAMP_PRECISION); + } } IFill[] previousFill = new IFill[inputColumns]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 7cdb51335ab..8d4a8b545ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -216,7 +216,9 @@ public class PlanGraphPrinter extends PlanVisitor<List<String>, PlanGraphPrinter boxValue.add(String.format("Fill-%s", node.getPlanNodeId().getId())); boxValue.add(String.format("Policy: %s", node.getFillDescriptor().getFillPolicy())); if (node.getFillDescriptor().getTimeDurationThreshold() != null) { - boxValue.add(String.format("TimeDurationThreshold: %s", node.getFillDescriptor().getTimeDurationThreshold())); + boxValue.add( + String.format( + "TimeDurationThreshold: %s", node.getFillDescriptor().getTimeDurationThreshold())); } return render(node, boxValue, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/FillDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/FillDescriptor.java index 20d08c784d6..8d3de2cfbe2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/FillDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/FillDescriptor.java @@ -40,7 +40,8 @@ public class FillDescriptor { // if private final TimeDuration timeDurationThreshold; - public FillDescriptor(FillPolicy fillPolicy, Literal fillValue, TimeDuration timeDurationThreshold) { + public FillDescriptor( + FillPolicy fillPolicy, Literal fillValue, TimeDuration timeDurationThreshold) { this.fillPolicy = fillPolicy; this.fillValue = fillValue; this.timeDurationThreshold = timeDurationThreshold; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/TestPlanBuilder.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/TestPlanBuilder.java index 36043c2b03d..eda0995d7c0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/TestPlanBuilder.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/TestPlanBuilder.java @@ -128,7 +128,11 @@ public class TestPlanBuilder { public TestPlanBuilder fill(String id, FillPolicy fillPolicy) { this.root = - new FillNode(new PlanNodeId(id), getRoot(), new FillDescriptor(fillPolicy, null, null), Ordering.ASC); + new FillNode( + new PlanNodeId(id), + getRoot(), + new FillDescriptor(fillPolicy, null, null), + Ordering.ASC); return this; } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java index 2ba4d4ca9d7..9e3d5d20581 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java @@ -194,9 +194,9 @@ public class TimeDuration implements Serializable { @Override public String toString() { - return "TimeDuration{" + - (monthDuration > 0 ? monthDuration + "mo, " : "") + - (nonMonthDuration > 0 ? nonMonthDuration : "")+ - '}'; + return "TimeDuration{" + + (monthDuration > 0 ? monthDuration + "mo, " : "") + + (nonMonthDuration > 0 ? nonMonthDuration : "") + + '}'; } }
