This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch optimize_series_reader
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/optimize_series_reader by this
push:
new 17365a7 remove hasOverlappedPage
17365a7 is described below
commit 17365a75a742d4095de6ea677eb37d35a7b6f7d3
Author: qiaojialin <[email protected]>
AuthorDate: Sun Mar 1 19:06:04 2020 +0800
remove hasOverlappedPage
---
.../groupby/GroupByWithoutValueFilterDataSet.java | 6 +-
.../db/query/executor/AggregationExecutor.java | 11 ++-
.../db/query/reader/series/IAggregateReader.java | 6 +-
.../query/reader/series/SeriesAggregateReader.java | 13 ++--
.../reader/series/SeriesRawDataBatchReader.java | 26 +------
.../iotdb/db/query/reader/series/SeriesReader.java | 84 +++++++++++++---------
.../reader/series/SeriesReaderByTimestamp.java | 27 +------
.../reader/series/SeriesAggregateReaderTest.java | 4 +-
8 files changed, 75 insertions(+), 102 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index bf9c5e6..f756816 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -221,10 +221,8 @@ public class GroupByWithoutValueFilterDataSet extends
GroupByEngineDataSet {
}
reader.skipCurrentPage();
continue;
- }
- while (reader.hasNextOverlappedPage()) {
- // cal by page data
- BatchData batchData = reader.nextOverlappedPage();
+ } else {
+ BatchData batchData = reader.nextPage();
for (int i = 0; i < aggregateResultList.size(); i++) {
if (Boolean.FALSE.equals(isCalculatedList.get(i))) {
AggregateResult result = aggregateResultList.get(i);
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
index c39a392..df3a164 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
@@ -173,15 +173,14 @@ public class AggregationExecutor {
}
seriesReader.skipCurrentPage();
continue;
- }
- // cal by page data
- while (seriesReader.hasNextOverlappedPage()) {
- BatchData nextOverlappedPageData = seriesReader.nextOverlappedPage();
+ } else {
+ // cal by page data
+ BatchData batchData = seriesReader.nextPage();
for (int i = 0; i < aggregateResultList.size(); i++) {
if (Boolean.FALSE.equals(isCalculatedList.get(i))) {
AggregateResult aggregateResult = aggregateResultList.get(i);
- aggregateResult.updateResultFromPageData(nextOverlappedPageData);
- nextOverlappedPageData.resetBatchData();
+ aggregateResult.updateResultFromPageData(batchData);
+ batchData.resetBatchData();
if (aggregateResult.isCalculatedAggregationResult()) {
isCalculatedList.set(i, true);
remainingToCalculate--;
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/IAggregateReader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/IAggregateReader.java
index bd22f3e..691a02a 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/IAggregateReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/IAggregateReader.java
@@ -47,7 +47,9 @@ public interface IAggregateReader {
void skipCurrentPage();
- boolean hasNextOverlappedPage() throws IOException;
+ BatchData nextPage() throws IOException;
- BatchData nextOverlappedPage() throws IOException;
+// boolean hasNextOverlappedPage() throws IOException;
+
+// BatchData nextOverlappedPage() throws IOException;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
index d711f02..453d6af 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReader.java
@@ -70,6 +70,9 @@ public class SeriesAggregateReader implements
IAggregateReader {
@Override
public boolean canUseCurrentPageStatistics() throws IOException {
Statistics currentPageStatistics = currentPageStatistics();
+ if (currentPageStatistics == null) {
+ return false;
+ }
return !seriesReader.isPageOverlapped() &&
containedByTimeFilter(currentPageStatistics);
}
@@ -84,16 +87,10 @@ public class SeriesAggregateReader implements
IAggregateReader {
}
@Override
- public boolean hasNextOverlappedPage() throws IOException {
- return seriesReader.hasNextOverlappedPage();
+ public BatchData nextPage() throws IOException {
+ return seriesReader.nextPage();
}
- @Override
- public BatchData nextOverlappedPage() throws IOException {
- return seriesReader.nextOverlappedPage();
- }
-
-
private boolean containedByTimeFilter(Statistics statistics) {
Filter timeFilter = seriesReader.getTimeFilter();
return timeFilter == null
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
index 553fdf8..19f3810 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
@@ -69,18 +69,14 @@ public class SeriesRawDataBatchReader implements
ManagedSeriesReader {
if (hasCachedBatchData) {
return true;
}
- /*
- * consume overlapped data firstly
- */
- if (readOverlappedPage()) {
- return hasCachedBatchData = true;
- }
+
/*
* consume page data secondly
*/
if (readPageData()) {
return hasCachedBatchData = true;
}
+
/*
* consume next chunk finally
*/
@@ -130,23 +126,7 @@ public class SeriesRawDataBatchReader implements
ManagedSeriesReader {
private boolean readPageData() throws IOException {
while (seriesReader.hasNextPage()) {
- if (!seriesReader.isPageOverlapped()) {
- batchData = seriesReader.nextPage();
- if (!isEmpty(batchData)) {
- return true;
- }
- continue;
- }
- if (readOverlappedPage()) {
- return true;
- }
- }
- return false;
- }
-
- private boolean readOverlappedPage() throws IOException {
- while (seriesReader.hasNextOverlappedPage()) {
- batchData = seriesReader.nextOverlappedPage();
+ batchData = seriesReader.nextPage();
if (!isEmpty(batchData)) {
return true;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index ddd4b4e..aff5159 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -90,7 +90,7 @@ public class SeriesReader {
/*
* result cache
*/
- private boolean hasCachedNextBatch;
+ private boolean hasCachedNextOverlappedPage;
private BatchData cachedBatchData;
public SeriesReader(Path seriesPath, TSDataType dataType, QueryContext
context,
@@ -179,6 +179,18 @@ public class SeriesReader {
}
}
+ if (firstPageReader != null && isPageOverlapped()) {
+ /*
+ * next page is overlapped, read data and cache
+ */
+ while (hasNextOverlappedPage()) {
+ cachedBatchData = nextOverlappedPage();
+ if (cachedBatchData != null && cachedBatchData.hasCurrent()) {
+ return hasCachedNextOverlappedPage = true;
+ }
+ }
+ }
+
return firstPageReader != null;
}
@@ -214,9 +226,9 @@ public class SeriesReader {
/**
* This method should be called after calling hasNextPage.
*/
- protected boolean isPageOverlapped() throws IOException {
- if (firstPageReader == null) {
- throw new IOException("no next page, make sure hasNextPage() is true");
+ protected boolean isPageOverlapped() {
+ if (hasCachedNextOverlappedPage) {
+ return true;
}
Statistics firstPageStatistics = firstPageReader.data.getStatistics();
@@ -230,37 +242,43 @@ public class SeriesReader {
* This method should only be used when the method isPageOverlapped() return
true.
*/
protected BatchData nextPage() throws IOException {
- if (isPageOverlapped()) {
- throw new IOException("next page is overlapped, make sure
isPageOverlapped is false");
- }
- BatchData pageData = firstPageReader.data.getAllSatisfiedPageData();
- firstPageReader = null;
+ if (hasCachedNextOverlappedPage) {
+ hasCachedNextOverlappedPage = false;
+ return cachedBatchData;
+ } else {
+ /*
+ * next page is not overlapped
+ */
+ BatchData pageData = firstPageReader.data.getAllSatisfiedPageData();
+ firstPageReader = null;
- /*
- * no value filter
- * only need to consider valueFilter because timeFilter has been set into
the page reader
- */
- if (valueFilter == null) {
- return pageData;
- }
+ /*
+ * no value filter
+ * only need to consider valueFilter because timeFilter has been set
into the page reader
+ */
+ if (valueFilter == null) {
+ return pageData;
+ }
- /*
- * has value filter
- */
- BatchData batchData = new BatchData(pageData.getDataType());
- while (pageData.hasCurrent()) {
- if (valueFilter.satisfy(pageData.currentTime(),
pageData.currentValue())) {
- batchData.putAnObject(pageData.currentTime(), pageData.currentValue());
+ /*
+ * has value filter
+ */
+ BatchData batchData = new BatchData(pageData.getDataType());
+ while (pageData.hasCurrent()) {
+ if (valueFilter.satisfy(pageData.currentTime(),
pageData.currentValue())) {
+ batchData.putAnObject(pageData.currentTime(),
pageData.currentValue());
+ }
+ pageData.next();
}
- pageData.next();
+ return batchData;
}
- return batchData;
+
}
public Statistics currentPageStatistics() throws IOException {
if (firstPageReader == null) {
- throw new IOException("No next page.");
+ return null;
}
return firstPageReader.data.getStatistics();
}
@@ -272,9 +290,9 @@ public class SeriesReader {
/**
* This method should be called after hasNextChunk and hasNextPage methods.
*/
- public boolean hasNextOverlappedPage() throws IOException {
+ private boolean hasNextOverlappedPage() throws IOException {
- if (hasCachedNextBatch) {
+ if (hasCachedNextOverlappedPage) {
return true;
}
@@ -305,9 +323,9 @@ public class SeriesReader {
mergeReader.nextTimeValuePair();
}
- hasCachedNextBatch = cachedBatchData.hasCurrent();
+ hasCachedNextOverlappedPage = cachedBatchData.hasCurrent();
}
- return hasCachedNextBatch;
+ return hasCachedNextOverlappedPage;
}
private void tryToPutAllDirectlyOverlappedPageReadersIntoMergeReader()
throws IOException {
@@ -357,9 +375,9 @@ public class SeriesReader {
pageReader.version, pageReader.data.getStatistics().getEndTime());
}
- public BatchData nextOverlappedPage() throws IOException {
- if (hasCachedNextBatch || hasNextOverlappedPage()) {
- hasCachedNextBatch = false;
+ private BatchData nextOverlappedPage() throws IOException {
+ if (hasCachedNextOverlappedPage || hasNextOverlappedPage()) {
+ hasCachedNextOverlappedPage = false;
return cachedBatchData;
}
throw new IOException("No more batch data");
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
index d5eb43c..70fb748 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
@@ -63,21 +63,14 @@ public class SeriesReaderByTimestamp implements
IReaderByTimestamp {
private boolean hasNext(long timestamp) throws IOException {
/*
- * consume overlapped data firstly
- */
- if (readOverlappedPage(timestamp)) {
- return true;
- }
-
- /*
- * consume pages secondly
+ * consume pages firstly
*/
if (readPageData(timestamp)) {
return true;
}
/*
- * consume next chunk
+ * consume chunk secondly
*/
while (seriesReader.hasNextChunk()) {
if (readPageData(timestamp)) {
@@ -93,23 +86,9 @@ public class SeriesReaderByTimestamp implements
IReaderByTimestamp {
if (!satisfyTimeFilter(seriesReader.currentPageStatistics())) {
seriesReader.skipCurrentPage();
continue;
- } else {
- batchData = seriesReader.nextPage();
- if (batchData.getTimeByIndex(batchData.length() - 1) >= timestamp) {
- return true;
- }
}
}
- if (readOverlappedPage(timestamp)) {
- return true;
- }
- }
- return false;
- }
-
- private boolean readOverlappedPage(long timestamp) throws IOException {
- while (seriesReader.hasNextOverlappedPage()) {
- batchData = seriesReader.nextOverlappedPage();
+ batchData = seriesReader.nextPage();
if (isEmpty(batchData)) {
continue;
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReaderTest.java
b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReaderTest.java
index 55d368b..ee85b15 100644
---
a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReaderTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesAggregateReaderTest.java
@@ -98,8 +98,8 @@ public class SeriesAggregateReaderTest {
assertEquals((long) loopTime * 20 + 60,
aggregateResult.getResult());
}
- while (seriesReader.hasNextOverlappedPage()) {
- BatchData nextOverlappedPageData =
seriesReader.nextOverlappedPage();
+ while (seriesReader.hasNextPage()) {
+ BatchData nextOverlappedPageData = seriesReader.nextPage();
aggregateResult.updateResultFromPageData(nextOverlappedPageData);
nextOverlappedPageData.resetBatchData();
assertEquals(true, nextOverlappedPageData.hasCurrent());
