This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f98b3d218e0 Fix LIMIT/OFFSET push down in GROUP BY MONTH
f98b3d218e0 is described below
commit f98b3d218e0bcc94f8277ce4db51525fa49ea8b7
Author: YangCaiyin <[email protected]>
AuthorDate: Mon Dec 18 19:20:22 2023 +0800
Fix LIMIT/OFFSET push down in GROUP BY MONTH
---
.../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 57 +++++++++++++
.../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 44 ++++++----
.../IoTDBGroupByNaturalMonthNsPrecisionIT.java | 12 +--
.../IoTDBGroupByNaturalMonthUsPrecisionIT.java | 12 +--
.../iotdb/db/protocol/session/SessionManager.java | 2 +-
.../timerangeiterator/AggrWindowIterator.java | 39 ++++++---
.../plan/optimization/LimitOffsetPushDown.java | 59 +++++++++++++-
.../plan/planner/LogicalPlanBuilder.java | 11 ++-
.../plan/planner/LogicalPlanVisitor.java | 3 +-
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 12 +--
.../aggregation/TimeRangeIteratorTest.java | 66 +++++++--------
.../read/filter/operator/GroupByMonthFilter.java | 11 ++-
.../apache/iotdb/tsfile/utils/TimeDuration.java | 93 ++++++++++------------
.../iotdb/tsfile/utils/TimeDurationTest.java | 40 ++++++----
14 files changed, 299 insertions(+), 162 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IOTDBGroupByIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IOTDBGroupByIT.java
index 7fa8fdc3f81..1c9d5dfa6ca 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IOTDBGroupByIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IOTDBGroupByIT.java
@@ -733,4 +733,61 @@ public class IOTDBGroupByIT {
expectedHeader,
retArray);
}
+
+ @Test
+ public void endTimeGroupByTimeTest8() {
+ String[] expectedHeader =
+ new String[] {
+ TIMESTAMP_STR,
+ "Device",
+ END_TIMESTAMP_STR,
+ count("temperature"),
+ sum("temperature"),
+ avg("temperature")
+ };
+ String[] retArray =
+ new String[] {
+ "15,root.ln.wf01.wt01,19,0,null,null,",
+ "20,root.ln.wf01.wt01,24,1,20.2,20.2,",
+ "25,root.ln.wf01.wt01,29,0,null,null,",
+ "30,root.ln.wf01.wt01,34,1,30.3,30.3,",
+ "35,root.ln.wf01.wt01,39,0,null,null,"
+ };
+ resultSetEqualTest(
+ "select __endTime,count(temperature), sum(temperature),
avg(temperature) from "
+ + "root.ln.wf01.wt01 "
+ + "WHERE temperature > 0 "
+ + "GROUP BY ([0, 600), 5ms) "
+ + "limit 5 offset 3 align by device",
+ expectedHeader,
+ retArray);
+ }
+
+ @Test
+ public void endTimeGroupByTimeTest9() {
+ String[] expectedHeader =
+ new String[] {
+ TIMESTAMP_STR,
+ "Device",
+ END_TIMESTAMP_STR,
+ count("temperature"),
+ sum("temperature"),
+ avg("temperature")
+ };
+ String[] retArray =
+ new String[] {
+ "45,root.ln.wf01.wt01,45,3,90.9,30.3,",
+ "50,root.ln.wf01.wt01,50,3,121.2,40.4,",
+ "55,root.ln.wf01.wt01,55,3,121.2,40.4,",
+ "60,root.ln.wf01.wt01,60,2,90.9,45.45,",
+ "65,root.ln.wf01.wt01,65,2,90.9,45.45,"
+ };
+ resultSetEqualTest(
+ "select __endTime,count(temperature), sum(temperature),
avg(temperature) from "
+ + "root.ln.wf01.wt01 "
+ + "GROUP BY ((0, 600], 30ms, 5ms) "
+ + "limit 5 offset 3 align by device",
+ expectedHeader,
+ retArray);
+ }
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
index dc33dd9574a..e08eb4c5f1d 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
@@ -171,6 +171,20 @@ public class IoTDBGroupByNaturalMonthIT {
currPrecision);
}
+ @Test
+ public void groupByNaturalMonthTest5() {
+ String[] expectedHeader = new String[] {TIMESTAMP_STR,
sum("root.sg1.d1.temperature")};
+ String[] retArray = {
+ "01/30/2021:00:00:00,29.0,", "02/28/2021:00:00:00,30.0,",
"03/30/2021:00:00:00,1.0,"
+ };
+ resultSetEqualTest(
+ "select sum(temperature) from root.sg1.d1 GROUP BY ([2021-01-30,
2021-03-31), 1mo)",
+ expectedHeader,
+ retArray,
+ df,
+ currPrecision);
+ }
+
/** Test group by month with order by time desc. */
@Test
public void groupByNaturalMonthFailTest() {
@@ -241,12 +255,12 @@ public class IoTDBGroupByNaturalMonthIT {
"10/31/2020:00:00:00,30.0,",
"11/10/2020:00:00:00,30.0,",
"11/20/2020:00:00:00,30.0,",
- "11/30/2020:00:00:00,31.0,",
- "12/10/2020:00:00:00,31.0,",
- "12/20/2020:00:00:00,31.0,",
- "12/30/2020:00:00:00,31.0,",
- "01/09/2021:00:00:00,31.0,",
- "01/19/2021:00:00:00,31.0,",
+ "11/30/2020:00:00:00,30.0,",
+ "12/10/2020:00:00:00,30.0,",
+ "12/20/2020:00:00:00,30.0,",
+ "12/30/2020:00:00:00,30.0,",
+ "01/09/2021:00:00:00,30.0,",
+ "01/19/2021:00:00:00,30.0,",
"01/29/2021:00:00:00,30.0,",
"02/08/2021:00:00:00,21.0,",
"02/18/2021:00:00:00,11.0,",
@@ -363,11 +377,11 @@ public class IoTDBGroupByNaturalMonthIT {
// [01-28, 03-01)
"01/28/2023:00:00:00,1,",
// [03-01, 04-02)
- "03/01/2023:00:00:00,2,",
+ "03/01/2023:00:00:00,1,",
// [04-02, 05-03)
- "04/02/2023:00:00:00,1,",
+ "03/30/2023:00:00:00,1,",
// [05-03, 05-29)
- "05/03/2023:00:00:00,0,"
+ "05/01/2023:00:00:00,1,"
};
resultSetEqualTest(
"select count(s1) from root.test.d1 " + "group by ([2023-01-28,
2023-05-29), 1mo1d)",
@@ -384,12 +398,12 @@ public class IoTDBGroupByNaturalMonthIT {
new String[] {
// [01-28, 03-01)
"1674864000000,1,",
- // [03-01, 04-02)
- "1677628800000,2,",
- // [04-02, 05-03)
- "1680393600000,1,",
- // [05-03, 05-29)
- "1683072000000,0,"
+ // [03-01, 03-30)
+ "1677628800000,1,",
+ // [03-30, 05-01)
+ "1680134400000,1,",
+ // [05-01, 05-29)
+ "1682899200000,1,"
};
// the part in timeDuration finer than current time precision will be
discarded
resultSetEqualTest(
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthNsPrecisionIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthNsPrecisionIT.java
index 8ff16207304..807ecf6620e 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthNsPrecisionIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthNsPrecisionIT.java
@@ -89,12 +89,12 @@ public class IoTDBGroupByNaturalMonthNsPrecisionIT extends
IoTDBGroupByNaturalMo
new String[] {
// [01-28, 03-01 + 1ns)
"1674864000000000000,2,",
- // [03-01 + 1ns, 04-02 + 2ns)
- "1677628800000000001,1,",
- // [04-02 + 2ns, 05-03 + 3ns)
- "1680393600000000002,1,",
- // [05-03 + 3ns, 05-29 + 4ns)
- "1683072000000000003,0,"
+ // [03-01 + 1ns, 03-30 + 2ns)
+ "1677628800000000001,0,",
+ // [03-30 + 2ns, 05-01 + 3ns)
+ "1680134400000000002,2,",
+ // [05-01 + 3ns, 05-29 + 4ns)
+ "1682899200000000003,0,"
};
// the part in timeDuration finer than current time precision will be
discarded
resultSetEqualTest(
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthUsPrecisionIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthUsPrecisionIT.java
index 0f55af25da8..eb4bfec938f 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthUsPrecisionIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthUsPrecisionIT.java
@@ -74,12 +74,12 @@ public class IoTDBGroupByNaturalMonthUsPrecisionIT extends
IoTDBGroupByNaturalMo
new String[] {
// [01-28, 03-01)
"1674864000000000,1,",
- // [03-01, 04-02)
- "1677628800000000,2,",
- // [04-02, 05-03)
- "1680393600000000,1,",
- // [05-03, 05-29)
- "1683072000000000,0,"
+ // [03-01, 03-30)
+ "1677628800000000,1,",
+ // [03-30, 05-01)
+ "1680134400000000,1,",
+ // [05-01, 05-29)
+ "1682899200000000,1,"
};
// the part in timeDuration finer than current time precision will be
discarded
resultSetEqualTest(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
index eafec00cd8d..3b1a4dc5e71 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
@@ -296,7 +296,7 @@ public class SessionManager implements SessionManagerMBean {
return session.getTimeZone();
} else {
// only used for test
- return TimeZone.getTimeZone("+08:00");
+ return TimeZone.getTimeZone(ZoneId.systemDefault());
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java
index 25264d58e4f..f370815cb6e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java
@@ -42,6 +42,8 @@ public class AggrWindowIterator implements ITimeRangeIterator
{
private TimeRange curTimeRange;
private boolean hasCachedTimeRange;
+ // The number of current timeRange, it's used to calculate the cpu when
there contains month
+ private int timeRangeCount;
@SuppressWarnings("squid:S107")
public AggrWindowIterator(
@@ -57,6 +59,7 @@ public class AggrWindowIterator implements ITimeRangeIterator
{
this.slidingStep = slidingStep;
this.isAscending = isAscending;
this.leftCRightO = leftCRightO;
+ this.timeRangeCount = 0;
}
@Override
@@ -74,7 +77,7 @@ public class AggrWindowIterator implements ITimeRangeIterator
{
// calculate interval length by natural month based on startTime
// ie. startTIme = 1/31, interval = 1mo, curEndTime will be set to 2/29
retEndTime =
- Math.min(DateTimeUtils.calcPositiveIntervalByMonth(startTime,
interval, 1), endTime);
+ Math.min(DateTimeUtils.calcPositiveIntervalByMonth(startTime,
interval), endTime);
} else {
retEndTime = Math.min(startTime + interval.nonMonthDuration, endTime);
}
@@ -94,12 +97,17 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
(double) queryRange
/
(slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)));
long tempRetStartTime =
- DateTimeUtils.calcPositiveIntervalByMonth(startTime, slidingStep,
intervalNum - 1);
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ startTime, slidingStep.multiple(intervalNum - 1));
retStartTime = tempRetStartTime;
while (tempRetStartTime < endTime) {
+ intervalNum++;
retStartTime = tempRetStartTime;
- tempRetStartTime =
DateTimeUtils.calcPositiveIntervalByMonth(retStartTime, slidingStep, 1);
+ tempRetStartTime =
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ retStartTime, slidingStep.multiple(intervalNum - 1));
}
+ intervalNum -= 1;
} else {
intervalNum = (long) Math.ceil(queryRange / (double)
slidingStep.nonMonthDuration);
retStartTime = slidingStep.nonMonthDuration * (intervalNum - 1) +
startTime;
@@ -109,7 +117,10 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
// calculate interval length by natural month based on curStartTime
// ie. startTIme = 1/31, interval = 1mo, curEndTime will be set to 2/29
retEndTime =
- Math.min(DateTimeUtils.calcPositiveIntervalByMonth(retStartTime,
interval, 1), endTime);
+ Math.min(
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ startTime, interval.merge(slidingStep.multiple(intervalNum -
1))),
+ endTime);
} else {
retEndTime = Math.min(retStartTime + interval.nonMonthDuration, endTime);
}
@@ -123,6 +134,7 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
}
if (curTimeRange == null) {
curTimeRange = getFirstTimeRange();
+ timeRangeCount++;
hasCachedTimeRange = true;
return true;
}
@@ -132,7 +144,9 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
long curStartTime = curTimeRange.getMin();
if (isAscending) {
if (slidingStep.containsMonth()) {
- retStartTime = DateTimeUtils.calcPositiveIntervalByMonth(curStartTime,
slidingStep, 1);
+ retStartTime =
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ startTime, slidingStep.multiple(timeRangeCount));
} else {
retStartTime = curStartTime + slidingStep.nonMonthDuration;
}
@@ -142,7 +156,9 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
}
} else {
if (slidingStep.containsMonth()) {
- retStartTime = DateTimeUtils.calcNegativeIntervalByMonth(curStartTime,
slidingStep);
+ // group by month doesn't support ascending.
+ throw new UnsupportedOperationException(
+ "Ascending is not supported when sliding step contains month.");
} else {
retStartTime = curStartTime - slidingStep.nonMonthDuration;
}
@@ -152,13 +168,16 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
}
if (interval.containsMonth()) {
- retEndTime = DateTimeUtils.calcPositiveIntervalByMonth(retStartTime,
interval, 1);
+ retEndTime =
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ startTime, slidingStep.multiple(timeRangeCount).merge(interval));
} else {
retEndTime = retStartTime + interval.nonMonthDuration;
}
retEndTime = Math.min(retEndTime, endTime);
curTimeRange = new TimeRange(retStartTime, retEndTime);
hasCachedTimeRange = true;
+ timeRangeCount++;
return true;
}
@@ -193,10 +212,11 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
(double) queryRange
/
(slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)));
long retStartTime =
- DateTimeUtils.calcPositiveIntervalByMonth(startTime, slidingStep,
intervalNum);
+ DateTimeUtils.calcPositiveIntervalByMonth(startTime,
slidingStep.multiple(intervalNum));
while (retStartTime < endTime) {
intervalNum++;
- retStartTime = DateTimeUtils.calcPositiveIntervalByMonth(retStartTime,
slidingStep, 1);
+ retStartTime =
+ DateTimeUtils.calcPositiveIntervalByMonth(startTime,
slidingStep.multiple(intervalNum));
}
} else {
intervalNum = (long) Math.ceil(queryRange / (double)
slidingStep.nonMonthDuration);
@@ -207,5 +227,6 @@ public class AggrWindowIterator implements
ITimeRangeIterator {
public void reset() {
curTimeRange = null;
hasCachedTimeRange = false;
+ timeRangeCount = 0;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
index b5081523108..52f7e53fbcb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java
@@ -41,10 +41,12 @@ import
org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy;
import
org.apache.iotdb.db.queryengine.plan.statement.component.GroupByTimeComponent;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
+import org.apache.iotdb.db.utils.DateTimeUtils;
+import org.apache.iotdb.tsfile.utils.TimeDuration;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* <b>Optimization phase:</b> Distributed plan planning
@@ -261,8 +263,61 @@ public class LimitOffsetPushDown implements PlanOptimizer {
return false;
}
+ private static void pushDownLimitOffsetToTimeParameterContainingMonth(
+ QueryStatement queryStatement) {
+ GroupByTimeComponent groupByTimeComponent =
queryStatement.getGroupByTimeComponent();
+ long startTime = groupByTimeComponent.getStartTime();
+ long endTime = groupByTimeComponent.getEndTime();
+ TimeDuration slidingStep = groupByTimeComponent.getSlidingStep();
+ TimeDuration interval = groupByTimeComponent.getInterval();
+ long limitSize = queryStatement.getRowLimit();
+ long offsetSize = queryStatement.getRowOffset();
+
+ // Evaluate the day of month as 28 days
+ long totalStep = slidingStep.getMinTotalDuration(TimeUnit.MILLISECONDS);
+ long size = (endTime - startTime + totalStep - 1) / totalStep;
+ if (size > offsetSize) {
+ // ordering in group by month must be ascending
+ long newStartTime =
+ DateTimeUtils.calcPositiveIntervalByMonth(startTime,
slidingStep.multiple(offsetSize));
+
+ if (limitSize != 0) {
+ endTime =
+ Math.min(
+ endTime,
+ DateTimeUtils.calcPositiveIntervalByMonth(
+ startTime,
+ calculateEndTimeDuration(slidingStep, interval, limitSize,
offsetSize)));
+ }
+ groupByTimeComponent.setEndTime(endTime);
+ groupByTimeComponent.setStartTime(newStartTime);
+ } else {
+ // finish the query, resultSet is empty
+ queryStatement.setResultSetEmpty(true);
+ }
+ // If windows overlap, we need to keep LIMIT because the window size can
be less than interval
+ // which may result in more windows than we need in the target time range.
+ queryStatement.setRowLimit(interval.isGreaterThan(slidingStep) ? limitSize
: 0);
+ queryStatement.setRowOffset(0);
+ }
+
+ private static TimeDuration calculateEndTimeDuration(
+ TimeDuration slidingStep, TimeDuration interval, long limitSize, long
offsetSize) {
+ long length = offsetSize + limitSize - 1;
+ // startTime + offsetSize * step + (limitSize - 1) * step + interval
+ int monthDuration = (int) (length * slidingStep.monthDuration +
interval.monthDuration);
+ long nonMonthDuration = length * slidingStep.nonMonthDuration +
interval.nonMonthDuration;
+ return new TimeDuration(monthDuration, nonMonthDuration);
+ }
+
public static void pushDownLimitOffsetToTimeParameter(QueryStatement
queryStatement) {
GroupByTimeComponent groupByTimeComponent =
queryStatement.getGroupByTimeComponent();
+ // if group by time contains month, we use another push down limit/offset
+ if (groupByTimeComponent.getInterval().containsMonth()
+ || groupByTimeComponent.getSlidingStep().containsMonth()) {
+ pushDownLimitOffsetToTimeParameterContainingMonth(queryStatement);
+ return;
+ }
long startTime = groupByTimeComponent.getStartTime();
long endTime = groupByTimeComponent.getEndTime();
long step = groupByTimeComponent.getSlidingStep().nonMonthDuration;
@@ -313,7 +368,7 @@ public class LimitOffsetPushDown implements PlanOptimizer {
GroupByTimeComponent groupByTimeComponent =
queryStatement.getGroupByTimeComponent();
if (groupByTimeComponent.getInterval().containsMonth()
|| groupByTimeComponent.getSlidingStep().containsMonth()) {
- return Collections.emptyList();
+ return deviceNames;
}
long startTime = groupByTimeComponent.getStartTime();
long endTime = groupByTimeComponent.getEndTime();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
index 597451b6def..047fab98504 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
@@ -450,9 +450,10 @@ public class LogicalPlanBuilder {
Set<Expression> aggregationExpressions,
Set<Expression> sourceTransformExpressions,
LinkedHashMap<Expression, Set<Expression>> crossGroupByExpressions,
- List<Integer> deviceViewInputIndexes) {
+ List<Integer> deviceViewInputIndexes,
+ boolean outputEndTime) {
checkArgument(
- aggregationExpressions.size() == deviceViewInputIndexes.size(),
+ aggregationExpressions.size() <= deviceViewInputIndexes.size(),
"Each aggregate should correspond to a column of output.");
boolean needCheckAscending = groupByTimeParameter == null;
@@ -461,7 +462,8 @@ public class LogicalPlanBuilder {
Map<AggregationDescriptor, Integer> aggregationToIndexMap = new
HashMap<>();
Map<PartialPath, List<AggregationDescriptor>> countTimeAggregations = new
HashMap<>();
- int index = 0;
+ // If need output endTime, the first index is used by __endTime
+ int index = outputEndTime ? 1 : 0;
for (Expression aggregationExpression : aggregationExpressions) {
AggregationDescriptor aggregationDescriptor =
createAggregationDescriptor(
@@ -489,6 +491,9 @@ public class LogicalPlanBuilder {
if (!curStep.isOutputPartial()) {
// update measurementIndexes
deviceViewInputIndexes.clear();
+ if (outputEndTime) {
+ deviceViewInputIndexes.add(1);
+ }
deviceViewInputIndexes.addAll(
sourceNodeList.stream()
.map(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
index 1ebb78ef36f..0b131538942 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
@@ -352,7 +352,8 @@ public class LogicalPlanVisitor extends
StatementVisitor<PlanNode, MPPQueryConte
aggregationExpressions,
sourceTransformExpressions,
analysis.getCrossGroupByExpressions(),
- deviceViewInputIndexes);
+ deviceViewInputIndexes,
+ queryStatement.isOutputEndTime());
}
if (queryStatement.isGroupByTime() && queryStatement.isOutputEndTime()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java
index 278122486d5..6c48e26445e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java
@@ -751,18 +751,8 @@ public class DateTimeUtils {
public static final long MS_TO_MONTH = 30 * 86400_000L;
- public static long calcPositiveIntervalByMonth(
- long startTime, TimeDuration duration, long times) {
+ public static long calcPositiveIntervalByMonth(long startTime, TimeDuration
duration) {
return TimeDuration.calcPositiveIntervalByMonth(
- startTime,
- duration,
- times,
- SessionManager.getInstance().getSessionTimeZone(),
- TimestampPrecisionUtils.currPrecision);
- }
-
- public static long calcNegativeIntervalByMonth(long startTime, TimeDuration
duration) {
- return TimeDuration.calcNegativeIntervalByMonth(
startTime,
duration,
SessionManager.getInstance().getSessionTimeZone(),
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java
index 592110d41fc..6a2db3bdd82 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java
@@ -236,16 +236,16 @@ public class TimeRangeIteratorTest {
"[ 1604102400000 : 1606694399999 ]",
"[ 1604966400000 : 1607558399999 ]",
"[ 1605830400000 : 1608422399999 ]",
- "[ 1606694400000 : 1609372799999 ]",
- "[ 1607558400000 : 1610236799999 ]",
- "[ 1608422400000 : 1611100799999 ]",
- "[ 1609286400000 : 1611964799999 ]",
- "[ 1610150400000 : 1612828799999 ]",
- "[ 1611014400000 : 1613692799999 ]",
+ "[ 1606694400000 : 1609286399999 ]",
+ "[ 1607558400000 : 1610150399999 ]",
+ "[ 1608422400000 : 1611014399999 ]",
+ "[ 1609286400000 : 1611878399999 ]",
+ "[ 1610150400000 : 1612742399999 ]",
+ "[ 1611014400000 : 1613606399999 ]",
"[ 1611878400000 : 1614470399999 ]",
- "[ 1612742400000 : 1615161599999 ]",
- "[ 1613606400000 : 1616025599999 ]",
- "[ 1614470400000 : 1617148799999 ]",
+ "[ 1612742400000 : 1615334399999 ]",
+ "[ 1613606400000 : 1616198399999 ]",
+ "[ 1614470400000 : 1617062399999 ]",
"[ 1615334400000 : 1617148799999 ]",
"[ 1616198400000 : 1617148799999 ]",
"[ 1617062400000 : 1617148799999 ]"
@@ -257,22 +257,14 @@ public class TimeRangeIteratorTest {
"[ 1606694400000 : 1607558399999 ]",
"[ 1607558400000 : 1608422399999 ]",
"[ 1608422400000 : 1609286399999 ]",
- "[ 1609286400000 : 1609372799999 ]",
- "[ 1609372800000 : 1610150399999 ]",
- "[ 1610150400000 : 1610236799999 ]",
- "[ 1610236800000 : 1611014399999 ]",
- "[ 1611014400000 : 1611100799999 ]",
- "[ 1611100800000 : 1611878399999 ]",
- "[ 1611878400000 : 1611964799999 ]",
- "[ 1611964800000 : 1612742399999 ]",
- "[ 1612742400000 : 1612828799999 ]",
- "[ 1612828800000 : 1613606399999 ]",
- "[ 1613606400000 : 1613692799999 ]",
- "[ 1613692800000 : 1614470399999 ]",
- "[ 1614470400000 : 1615161599999 ]",
- "[ 1615161600000 : 1615334399999 ]",
- "[ 1615334400000 : 1616025599999 ]",
- "[ 1616025600000 : 1616198399999 ]",
+ "[ 1609286400000 : 1610150399999 ]",
+ "[ 1610150400000 : 1611014399999 ]",
+ "[ 1611014400000 : 1611878399999 ]",
+ "[ 1611878400000 : 1612742399999 ]",
+ "[ 1612742400000 : 1613606399999 ]",
+ "[ 1613606400000 : 1614470399999 ]",
+ "[ 1614470400000 : 1615334399999 ]",
+ "[ 1615334400000 : 1616198399999 ]",
"[ 1616198400000 : 1617062399999 ]",
"[ 1617062400000 : 1617148799999 ]"
};
@@ -350,15 +342,15 @@ public class TimeRangeIteratorTest {
"[ "
+ Timestamp.valueOf("2023-03-01 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-04-02 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-03-30 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-04-02 00:00:00").getTime()
+ + Timestamp.valueOf("2023-03-30 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-05-03 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-05-01 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-05-03 00:00:00").getTime()
+ + Timestamp.valueOf("2023-05-01 00:00:00").getTime()
+ " : "
+ (Timestamp.valueOf("2023-05-29 00:00:00").getTime() - 1L)
+ " ]"
@@ -389,25 +381,25 @@ public class TimeRangeIteratorTest {
"[ "
+ Timestamp.valueOf("2023-03-02 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-04-02 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-03-30 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-04-02 00:00:00").getTime()
+ + Timestamp.valueOf("2023-03-30 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-04-03 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-03-31 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-04-03 00:00:00").getTime()
+ + Timestamp.valueOf("2023-03-31 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-05-03 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-05-01 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-05-03 00:00:00").getTime()
+ + Timestamp.valueOf("2023-05-01 00:00:00").getTime()
+ " : "
- + (Timestamp.valueOf("2023-05-04 00:00:00").getTime() - 1L)
+ + (Timestamp.valueOf("2023-05-02 00:00:00").getTime() - 1L)
+ " ]",
"[ "
- + Timestamp.valueOf("2023-05-04 00:00:00").getTime()
+ + Timestamp.valueOf("2023-05-02 00:00:00").getTime()
+ " : "
+ (Timestamp.valueOf("2023-05-29 00:00:00").getTime() - 1L)
+ " ]",
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GroupByMonthFilter.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GroupByMonthFilter.java
index 0c2a9895b6e..58cda989643 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GroupByMonthFilter.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GroupByMonthFilter.java
@@ -186,14 +186,19 @@ public class GroupByMonthFilter extends GroupByFilter {
}
this.startTime = startTimes[n];
this.slidingStep =
- calcPositiveIntervalByMonth(startTime, originalSlidingStep, 1,
timeZone, currPrecision)
+ calcPositiveIntervalByMonth(
+ originalStartTime, originalSlidingStep.multiple(n + 1),
timeZone, currPrecision)
- startTime;
} else {
- startTime = originalStartTime + n * slidingStep;
+ this.startTime = originalStartTime + n * slidingStep;
}
if (originalInterval.containsMonth()) {
this.interval =
- calcPositiveIntervalByMonth(startTime, originalInterval, 1,
timeZone, currPrecision)
+ calcPositiveIntervalByMonth(
+ originalStartTime,
+ originalSlidingStep.multiple(n).merge(originalInterval),
+ timeZone,
+ currPrecision)
- startTime;
}
}
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java
index 9e3d5d20581..8a0045ddf63 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java
@@ -22,7 +22,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.util.Calendar;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.Objects;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
@@ -59,6 +61,24 @@ public class TimeDuration implements Serializable {
+ nonMonthDuration;
}
+ public boolean isGreaterThan(TimeDuration right) {
+ if (this.monthDuration > right.monthDuration) {
+ return true;
+ } else if (this.monthDuration == right.monthDuration) {
+ return this.nonMonthDuration > right.nonMonthDuration;
+ }
+ return false;
+ }
+
+ public TimeDuration merge(TimeDuration other) {
+ return new TimeDuration(
+ this.monthDuration + other.monthDuration, this.nonMonthDuration +
other.nonMonthDuration);
+ }
+
+ public TimeDuration multiple(long times) {
+ return new TimeDuration((int) (monthDuration * times), nonMonthDuration *
times);
+ }
+
/** Think month as 28 days. */
public long getMinTotalDuration(TimeUnit currPrecision) {
return currPrecision.convert(monthDuration * 28 * 86400_000L,
TimeUnit.MILLISECONDS)
@@ -88,76 +108,47 @@ public class TimeDuration implements Serializable {
TimeUnit currPrecision) {
long[] result = new long[length];
result[0] = startTime;
- Calendar calendar = Calendar.getInstance();
- calendar.setTimeZone(timeZone);
for (int i = 1; i < length; i++) {
- result[i] = getStartTime(result[i - 1], duration, currPrecision,
calendar);
+ result[i] = getStartTime(startTime, duration.multiple(i), currPrecision,
timeZone.toZoneId());
}
return result;
}
/**
- * Add several time durations contains natural months based on the startTime
and avoid edge cases,
- * ie 2/28
+ * Add time duration contains natural months to startTime.
+ *
+ * <p>Attention: This method does not support accumulation. If you need to
calculate the date two
+ * months after the start time, just add two months directly through
duration, rather than adding
+ * one month first and then adding another month in a loop, it will get
wrong result.
+ *
+ * <p>There is an example:
+ *
+ * <pre>
+ * 1.30 + 2mo = 3.30(right)
+ * 1.30 + 1mo = 2.28, 2.28 + 1mo = 3.28(wrong)
+ * </pre>
*
* @param startTime start time
* @param duration one duration
- * @param times num of duration elapsed
* @return the time after durations elapsed
*/
public static long calcPositiveIntervalByMonth(
- long startTime,
- TimeDuration duration,
- long times,
- TimeZone timeZone,
- TimeUnit currPrecision) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTimeZone(timeZone);
- for (int i = 0; i < times; i++) {
- startTime = getStartTime(startTime, duration, currPrecision, calendar);
- }
- return startTime;
+ long startTime, TimeDuration duration, TimeZone timeZone, TimeUnit
currPrecision) {
+ return getStartTime(startTime, duration, currPrecision,
timeZone.toZoneId());
}
private static long getStartTime(
- long startTime, TimeDuration duration, TimeUnit currPrecision, Calendar
calendar) {
+ long startTime, TimeDuration duration, TimeUnit currPrecision, ZoneId
zoneId) {
long coarserThanMsPart = getCoarserThanMsPart(startTime, currPrecision);
- calendar.setTimeInMillis(coarserThanMsPart);
- boolean isLastDayOfMonth =
- calendar.get(Calendar.DAY_OF_MONTH) ==
calendar.getActualMaximum(Calendar.DAY_OF_MONTH);
- calendar.add(Calendar.MONTH, duration.monthDuration);
- if (isLastDayOfMonth) {
- calendar.set(Calendar.DAY_OF_MONTH,
calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
- }
- return currPrecision.convert(calendar.getTimeInMillis(),
TimeUnit.MILLISECONDS)
+ LocalDateTime localDateTime =
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(coarserThanMsPart),
zoneId);
+ localDateTime = localDateTime.plusMonths(duration.monthDuration);
+ return currPrecision.convert(
+ localDateTime.atZone(zoneId).toInstant().toEpochMilli(),
TimeUnit.MILLISECONDS)
+ getFinerThanMsPart(startTime, currPrecision)
+ duration.nonMonthDuration;
}
- /**
- * subtract time duration contains natural months based on the startTime
- *
- * @param startTime start time
- * @param duration the duration
- * @return the time before duration
- */
- public static long calcNegativeIntervalByMonth(
- long startTime, TimeDuration duration, TimeZone timeZone, TimeUnit
currPrecision) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTimeZone(timeZone);
- long timeBeforeMonthElapsedInMs =
- TimeUnit.MILLISECONDS.convert(startTime - duration.nonMonthDuration,
currPrecision);
- calendar.setTimeInMillis(timeBeforeMonthElapsedInMs);
- boolean isLastDayOfMonth =
- calendar.get(Calendar.DAY_OF_MONTH) ==
calendar.getActualMaximum(Calendar.DAY_OF_MONTH);
- calendar.add(Calendar.MONTH, -duration.monthDuration);
- if (isLastDayOfMonth) {
- calendar.set(Calendar.DAY_OF_MONTH,
calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
- }
- return currPrecision.convert(calendar.getTimeInMillis(),
TimeUnit.MILLISECONDS)
- + getFinerThanMsPart(startTime - duration.nonMonthDuration,
currPrecision);
- }
-
private static long getCoarserThanMsPart(long time, TimeUnit currPrecision) {
return TimeUnit.MILLISECONDS.convert(time, currPrecision);
}
diff --git
a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TimeDurationTest.java
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TimeDurationTest.java
index ef3154f9590..ce8d7c9acc9 100644
---
a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TimeDurationTest.java
+++
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TimeDurationTest.java
@@ -25,43 +25,49 @@ import java.sql.Timestamp;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
-import static
org.apache.iotdb.tsfile.utils.TimeDuration.calcNegativeIntervalByMonth;
import static
org.apache.iotdb.tsfile.utils.TimeDuration.calcPositiveIntervalByMonth;
public class TimeDurationTest {
@Test
public void calculateIntervalTest() {
- // 1mo1d duration after 2023-01-31
+ // 1mo duration after 2023-01-31
long result =
+ calcPositiveIntervalByMonth(
+ Timestamp.valueOf("2023-01-31 00:00:00").getTime(),
+ new TimeDuration(1, 0),
+ TimeZone.getDefault(),
+ TimeUnit.MILLISECONDS);
+ Assert.assertEquals(Timestamp.valueOf("2023-02-28 00:00:00").getTime(),
result);
+ result =
+ calcPositiveIntervalByMonth(
+ Timestamp.valueOf("2023-02-28 00:00:00").getTime(),
+ new TimeDuration(2, 0),
+ TimeZone.getDefault(),
+ TimeUnit.MILLISECONDS);
+ Assert.assertEquals(Timestamp.valueOf("2023-04-28 00:00:00").getTime(),
result);
+ result =
+ calcPositiveIntervalByMonth(
+ Timestamp.valueOf("2023-01-30 00:00:00").getTime(),
+ new TimeDuration(2, 0),
+ TimeZone.getDefault(),
+ TimeUnit.MILLISECONDS);
+ Assert.assertEquals(Timestamp.valueOf("2023-03-30 00:00:00").getTime(),
result);
+ // 1mo1d duration after 2023-01-31
+ result =
calcPositiveIntervalByMonth(
Timestamp.valueOf("2023-01-31 00:00:00").getTime(),
new TimeDuration(1, 86400_000),
- 1,
TimeZone.getDefault(),
TimeUnit.MILLISECONDS);
Assert.assertEquals(Timestamp.valueOf("2023-03-01 00:00:00").getTime(),
result);
- // 1mo1d duration before 2023-03-01
- result =
- calcNegativeIntervalByMonth(
- result, new TimeDuration(1, 86400_000), TimeZone.getDefault(),
TimeUnit.MILLISECONDS);
- Assert.assertEquals(Timestamp.valueOf("2023-01-31 00:00:00").getTime(),
result);
// 1mo1d1ns duration after 2023-01-31
result =
calcPositiveIntervalByMonth(
Timestamp.valueOf("2023-01-31 00:00:00").getTime() * 1000_000,
new TimeDuration(1, 86400_000_000_001L),
- 1,
TimeZone.getDefault(),
TimeUnit.NANOSECONDS);
Assert.assertEquals(Timestamp.valueOf("2023-03-01 00:00:00").getTime() *
1000_000 + 1, result);
- // 1mo1d1ns duration before 2023-03-01
- result =
- calcNegativeIntervalByMonth(
- result,
- new TimeDuration(1, 86400_000_000_001L),
- TimeZone.getDefault(),
- TimeUnit.NANOSECONDS);
- Assert.assertEquals(Timestamp.valueOf("2023-01-31 00:00:00").getTime() *
1000_000, result);
}
}
