This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new c602d69f0a [To rel/0.13] [IOTDB-3645] Fix use statistics bug in
aggregation query (#6429)
c602d69f0a is described below
commit c602d69f0af9b34330425de4f0ff259adb7686d1
Author: liuminghui233 <[email protected]>
AuthorDate: Sat Jun 25 10:05:42 2022 +0800
[To rel/0.13] [IOTDB-3645] Fix use statistics bug in aggregation query
(#6429)
---
.../IoTDBGroupByQueryWithoutValueFilter2IT.java | 3 +-
...=> IoTDBGroupByQueryWithoutValueFilter3IT.java} | 7 +-
.../IoTDBGroupByQueryWithoutValueFilterIT.java | 43 ++++++++++++
.../groupby/LocalAlignedGroupByExecutor.java | 81 +++++++++++-----------
.../dataset/groupby/LocalGroupByExecutor.java | 56 +++++++--------
5 files changed, 117 insertions(+), 73 deletions(-)
diff --git
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
index 7f08a03329..674d4a4a4d 100644
---
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
+++
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
@@ -37,7 +37,6 @@ public class IoTDBGroupByQueryWithoutValueFilter2IT extends
IoTDBGroupByQueryWit
@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().initBeforeClass();
- // TODO When the aligned time series support compaction, we need to set
compaction to true
enableSeqSpaceCompaction =
IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction();
enableUnseqSpaceCompaction =
@@ -49,7 +48,7 @@ public class IoTDBGroupByQueryWithoutValueFilter2IT extends
IoTDBGroupByQueryWit
ConfigFactory.getConfig().setEnableSeqSpaceCompaction(false);
ConfigFactory.getConfig().setEnableUnseqSpaceCompaction(false);
ConfigFactory.getConfig().setEnableCrossSpaceCompaction(false);
- TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(3);
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(2);
AlignedWriteUtil.insertData();
}
diff --git
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter3IT.java
similarity index 89%
copy from
integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
copy to
integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter3IT.java
index 7f08a03329..193caa0911 100644
---
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter2IT.java
+++
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilter3IT.java
@@ -16,28 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.integration.aligned;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.integration.env.ConfigFactory;
import org.apache.iotdb.integration.env.EnvFactory;
-import org.apache.iotdb.itbase.category.LocalStandaloneTest;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-@Category({LocalStandaloneTest.class})
-public class IoTDBGroupByQueryWithoutValueFilter2IT extends
IoTDBGroupByQueryWithoutValueFilterIT {
+public class IoTDBGroupByQueryWithoutValueFilter3IT extends
IoTDBGroupByQueryWithoutValueFilterIT {
private static int numOfPointsPerPage;
@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().initBeforeClass();
- // TODO When the aligned time series support compaction, we need to set
compaction to true
enableSeqSpaceCompaction =
IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction();
enableUnseqSpaceCompaction =
diff --git
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilterIT.java
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilterIT.java
index a93a4771aa..02affb2fdd 100644
---
a/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilterIT.java
+++
b/integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBGroupByQueryWithoutValueFilterIT.java
@@ -1165,6 +1165,49 @@ public class IoTDBGroupByQueryWithoutValueFilterIT {
}
}
+ @Test
+ public void groupBySmallWindowTest() throws SQLException {
+ String[] retArray = new String[] {"10,0", "11,1", "12,1", "13,1", "14,1",
"15,1"};
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute("select count(s1) from root.sg1.d1 GROUP BY ([10,
16), 1ms)");
+ Assert.assertTrue(hasResultSet);
+
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(count("root.sg1.d1.s1"));
+ Assert.assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ Assert.assertEquals(retArray.length, cnt);
+ }
+
+ hasResultSet =
+ statement.execute(
+ "select count(s1) from root.sg1.d1 GROUP BY ([10, 16), 1ms)
order by time desc");
+ Assert.assertTrue(hasResultSet);
+
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = retArray.length;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString(count("root.sg1.d1.s1"));
+ Assert.assertEquals(retArray[cnt - 1], ans);
+ cnt--;
+ }
+ Assert.assertEquals(0, cnt);
+ }
+ }
+ }
+
@Test
public void groupByWithoutAggregationFuncTest() {
try (Connection connection = EnvFactory.getEnv().getConnection();
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalAlignedGroupByExecutor.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalAlignedGroupByExecutor.java
index 87faffebbb..fd00c3d3e3 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalAlignedGroupByExecutor.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalAlignedGroupByExecutor.java
@@ -124,27 +124,29 @@ public class LocalAlignedGroupByExecutor implements
AlignedGroupByExecutor {
// read from file
while (reader.hasNextFile()) {
// try to calc from fileMetaData
- Statistics fileTimeStatistics = reader.currentFileTimeStatistics();
- if (fileTimeStatistics.getStartTime() >= curEndTime) {
- if (ascending) {
- return results;
- } else {
+ if (reader.canUseCurrentFileStatistics()) {
+ Statistics fileTimeStatistics = reader.currentFileTimeStatistics();
+ if (fileTimeStatistics.getStartTime() >= curEndTime) {
+ if (ascending) {
+ return results;
+ } else {
+ reader.skipCurrentFile();
+ continue;
+ }
+ }
+ if (timeRange.contains(
+ fileTimeStatistics.getStartTime(),
fileTimeStatistics.getEndTime())) {
+ // calc from fileMetaData
+ while (reader.hasNextSubSeries()) {
+ Statistics currentFileStatistics = reader.currentFileStatistics();
+ calcFromStatistics(currentFileStatistics,
results.get(reader.getCurIndex()));
+ reader.nextSeries();
+ }
reader.skipCurrentFile();
continue;
}
}
- if (reader.canUseCurrentFileStatistics()
- && timeRange.contains(
- fileTimeStatistics.getStartTime(),
fileTimeStatistics.getEndTime())) {
- // calc from fileMetaData
- while (reader.hasNextSubSeries()) {
- Statistics currentFileStatistics = reader.currentFileStatistics();
- calcFromStatistics(currentFileStatistics,
results.get(reader.getCurIndex()));
- reader.nextSeries();
- }
- reader.skipCurrentFile();
- continue;
- }
+
// read chunk
if (readAndCalcFromChunk(curStartTime, curEndTime)) {
return results;
@@ -175,27 +177,29 @@ public class LocalAlignedGroupByExecutor implements
AlignedGroupByExecutor {
throws IOException, QueryProcessException {
while (reader.hasNextChunk()) {
// try to calc from chunkMetaData
- Statistics chunkTimeStatistics = reader.currentChunkTimeStatistics();
- if (chunkTimeStatistics.getStartTime() >= curEndTime) {
- if (ascending) {
- return true;
- } else {
+ if (reader.canUseCurrentChunkStatistics()) {
+ Statistics chunkTimeStatistics = reader.currentChunkTimeStatistics();
+ if (chunkTimeStatistics.getStartTime() >= curEndTime) {
+ if (ascending) {
+ return true;
+ } else {
+ reader.skipCurrentChunk();
+ continue;
+ }
+ }
+ if (timeRange.contains(
+ chunkTimeStatistics.getStartTime(),
chunkTimeStatistics.getEndTime())) {
+ // calc from chunkMetaData
+ while (reader.hasNextSubSeries()) {
+ Statistics currentChunkStatistics =
reader.currentChunkStatistics();
+ calcFromStatistics(currentChunkStatistics,
results.get(reader.getCurIndex()));
+ reader.nextSeries();
+ }
reader.skipCurrentChunk();
continue;
}
}
- if (reader.canUseCurrentChunkStatistics()
- && timeRange.contains(
- chunkTimeStatistics.getStartTime(),
chunkTimeStatistics.getEndTime())) {
- // calc from chunkMetaData
- while (reader.hasNextSubSeries()) {
- Statistics currentChunkStatistics = reader.currentChunkStatistics();
- calcFromStatistics(currentChunkStatistics,
results.get(reader.getCurIndex()));
- reader.nextSeries();
- }
- reader.skipCurrentChunk();
- continue;
- }
+
// read page
if (readAndCalcFromPage(curStartTime, curEndTime)) {
return true;
@@ -208,8 +212,8 @@ public class LocalAlignedGroupByExecutor implements
AlignedGroupByExecutor {
throws IOException, QueryProcessException {
while (reader.hasNextPage()) {
// try to calc from pageHeader
- Statistics pageTimeStatistics = reader.currentPageTimeStatistics();
- if (pageTimeStatistics != null) {
+ if (reader.canUseCurrentPageStatistics()) {
+ Statistics pageTimeStatistics = reader.currentPageTimeStatistics();
// current page max than time range
if (pageTimeStatistics.getStartTime() >= curEndTime) {
if (ascending) {
@@ -219,9 +223,8 @@ public class LocalAlignedGroupByExecutor implements
AlignedGroupByExecutor {
continue;
}
}
- if (reader.canUseCurrentPageStatistics()
- && timeRange.contains(
- pageTimeStatistics.getStartTime(),
pageTimeStatistics.getEndTime())) {
+ if (timeRange.contains(
+ pageTimeStatistics.getStartTime(),
pageTimeStatistics.getEndTime())) {
// calc from pageHeader
while (reader.hasNextSubSeries()) {
int subIndex = reader.getCurIndex();
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
index fd692c410a..1b96765e21 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
@@ -212,22 +212,23 @@ public class LocalGroupByExecutor implements
GroupByExecutor {
// read from file first
while (reader.hasNextFile()) {
- Statistics fileStatistics = reader.currentFileStatistics();
- if (fileStatistics.getStartTime() >= curEndTime) {
- if (ascending) {
- return results;
- } else {
+ if (reader.canUseCurrentFileStatistics()) {
+ Statistics fileStatistics = reader.currentFileStatistics();
+ if (fileStatistics.getStartTime() >= curEndTime) {
+ if (ascending) {
+ return results;
+ } else {
+ reader.skipCurrentFile();
+ continue;
+ }
+ }
+ // calc from fileMetaData
+ if (timeRange.contains(fileStatistics.getStartTime(),
fileStatistics.getEndTime())) {
+ calcFromStatistics(fileStatistics);
reader.skipCurrentFile();
continue;
}
}
- // calc from fileMetaData
- if (reader.canUseCurrentFileStatistics()
- && timeRange.contains(fileStatistics.getStartTime(),
fileStatistics.getEndTime())) {
- calcFromStatistics(fileStatistics);
- reader.skipCurrentFile();
- continue;
- }
// read chunk
if (readAndCalcFromChunk(curStartTime, curEndTime)) {
@@ -281,22 +282,24 @@ public class LocalGroupByExecutor implements
GroupByExecutor {
private boolean readAndCalcFromChunk(long curStartTime, long curEndTime)
throws IOException, QueryProcessException {
while (reader.hasNextChunk()) {
- Statistics chunkStatistics = reader.currentChunkStatistics();
- if (chunkStatistics.getStartTime() >= curEndTime) {
- if (ascending) {
- return true;
- } else {
+ if (reader.canUseCurrentChunkStatistics()) {
+ Statistics chunkStatistics = reader.currentChunkStatistics();
+ if (chunkStatistics.getStartTime() >= curEndTime) {
+ if (ascending) {
+ return true;
+ } else {
+ reader.skipCurrentChunk();
+ continue;
+ }
+ }
+ // calc from chunkMetaData
+ if (timeRange.contains(chunkStatistics.getStartTime(),
chunkStatistics.getEndTime())) {
+ calcFromStatistics(chunkStatistics);
reader.skipCurrentChunk();
continue;
}
}
- // calc from chunkMetaData
- if (reader.canUseCurrentChunkStatistics()
- && timeRange.contains(chunkStatistics.getStartTime(),
chunkStatistics.getEndTime())) {
- calcFromStatistics(chunkStatistics);
- reader.skipCurrentChunk();
- continue;
- }
+
// read page
if (readAndCalcFromPage(curStartTime, curEndTime)) {
return true;
@@ -309,9 +312,8 @@ public class LocalGroupByExecutor implements
GroupByExecutor {
private boolean readAndCalcFromPage(long curStartTime, long curEndTime)
throws IOException, QueryProcessException {
while (reader.hasNextPage()) {
- Statistics pageStatistics = reader.currentPageStatistics();
- // must be non overlapped page
- if (pageStatistics != null) {
+ if (reader.canUseCurrentPageStatistics()) {
+ Statistics pageStatistics = reader.currentPageStatistics();
// current page max than time range
if (pageStatistics.getStartTime() >= curEndTime) {
if (ascending) {
