This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty-opt in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 42d83cb9e8f9d9b3999e5a7e1c59724c890245ea Author: JackieTien97 <[email protected]> AuthorDate: Wed Sep 6 19:54:45 2023 +0800 try opt --- .../tsfile/read/reader/page/AlignedPageReader.java | 169 +++++++++++++-------- .../tsfile/read/reader/page/ValuePageReader.java | 38 +++++ .../read/reader/series/PaginationController.java | 8 + 3 files changed, 150 insertions(+), 65 deletions(-) diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java index 67be092ada5..e467d86bb22 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java @@ -167,93 +167,132 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { @Override public TsBlock getAllSatisfiedData() throws IOException { - builder.reset(); if (!pageSatisfy()) { return builder.build(); } long[] timeBatch = timePageReader.getNextTimeBatch(); - // if all the sub sensors' value are null in current row, just discard it - // if !filter.satisfy, discard this row - boolean[] keepCurrentRow = new boolean[timeBatch.length]; - if (filter == null) { - Arrays.fill(keepCurrentRow, true); + if (queryAllSensors && !isModified) { + // skip all the page + if (paginationController.hasCurOffset(timeBatch.length)) { + paginationController.consumeOffset(timeBatch.length); + } else { + // consume the remaining offset + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(paginationController.getCurOffset()); + } + int readStartIndex = + paginationController.hasCurOffset() ? (int) paginationController.getCurOffset() : 0; + // not included + int readEndIndex = + (paginationController.hasCurLimit() + && (paginationController.getCurLimit() < timeBatch.length - readStartIndex + 1)) + ? readStartIndex + (int) paginationController.getCurLimit() + : timeBatch.length; + if (paginationController.hasCurLimit()) { + paginationController.consumeLimit(readEndIndex - readStartIndex); + } + // construct time column + for (int i = readStartIndex; i < readEndIndex; i++) { + builder.getTimeColumnBuilder().writeLong(timeBatch[i]); + builder.declarePosition(); + } + // construct value columns + for (int i = 0; i < valueCount; i++) { + ValuePageReader pageReader = valuePageReaderList.get(i); + if (pageReader != null) { + pageReader.writeColumnBuilderWithNextBatch( + readStartIndex, readEndIndex, builder.getColumnBuilder(i)); + } else { + for (int j = readStartIndex; j < readEndIndex; j++) { + builder.getColumnBuilder(i).appendNull(); + } + } + } + } } else { - for (int i = 0, n = timeBatch.length; i < n; i++) { - keepCurrentRow[i] = filter.satisfy(timeBatch[i], null); + // if all the sub sensors' value are null in current row, just discard it + // if !filter.satisfy, discard this row + boolean[] keepCurrentRow = new boolean[timeBatch.length]; + if (filter == null) { + Arrays.fill(keepCurrentRow, true); + } else { + for (int i = 0, n = timeBatch.length; i < n; i++) { + keepCurrentRow[i] = filter.satisfy(timeBatch[i], null); + } } - } - boolean[][] isDeleted = null; - if (valueCount != 0) { - // using bitMap in valuePageReaders to indicate whether columns of current row are all null. - byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1]; - Arrays.fill(bitmask, (byte) 0x00); - isDeleted = new boolean[valueCount][timeBatch.length]; - for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) { - ValuePageReader pageReader = valuePageReaderList.get(columnIndex); - if (pageReader != null) { - byte[] bitmap = pageReader.getBitmap(); - pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]); + boolean[][] isDeleted = null; + if (valueCount != 0) { + // using bitMap in valuePageReaders to indicate whether columns of current row are all null. + byte[] bitmask = new byte[(timeBatch.length - 1) / 8 + 1]; + Arrays.fill(bitmask, (byte) 0x00); + isDeleted = new boolean[valueCount][timeBatch.length]; + for (int columnIndex = 0; columnIndex < valueCount; columnIndex++) { + ValuePageReader pageReader = valuePageReaderList.get(columnIndex); + if (pageReader != null) { + byte[] bitmap = pageReader.getBitmap(); + pageReader.fillIsDeleted(timeBatch, isDeleted[columnIndex]); - for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) { - if (isDeleted[columnIndex][i]) { - int shift = i % 8; - bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift))); + for (int i = 0, n = isDeleted[columnIndex].length; i < n; i++) { + if (isDeleted[columnIndex][i]) { + int shift = i % 8; + bitmap[i / 8] = (byte) (bitmap[i / 8] & (~(MASK >>> shift))); + } + } + for (int i = 0, n = bitmask.length; i < n; i++) { + bitmask[i] = (byte) (bitmap[i] | bitmask[i]); } - } - for (int i = 0, n = bitmask.length; i < n; i++) { - bitmask[i] = (byte) (bitmap[i] | bitmask[i]); } } - } - for (int i = 0, n = bitmask.length; i < n; i++) { - if (bitmask[i] == (byte) 0xFF) { - // 8 rows are not all null, do nothing - } else if (bitmask[i] == (byte) 0x00) { - for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) { - keepCurrentRow[i * 8 + j] = false; - } - } else { - for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) { - if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) { + for (int i = 0, n = bitmask.length; i < n; i++) { + if (bitmask[i] == (byte) 0xFF) { + // 8 rows are not all null, do nothing + } else if (bitmask[i] == (byte) 0x00) { + for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) { keepCurrentRow[i * 8 + j] = false; } + } else { + for (int j = 0; j < 8 && (i * 8 + j < keepCurrentRow.length); j++) { + if (((bitmask[i] & 0xFF) & (MASK >>> j)) == 0) { + keepCurrentRow[i * 8 + j] = false; + } + } } } } - } - // construct time column - int readEndIndex = timeBatch.length; - for (int i = 0; i < timeBatch.length; i++) { - if (keepCurrentRow[i]) { - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - keepCurrentRow[i] = false; - } else if (paginationController.hasCurLimit()) { - builder.getTimeColumnBuilder().writeLong(timeBatch[i]); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - readEndIndex = i; - break; + // construct time column + int readEndIndex = timeBatch.length; + for (int i = 0; i < timeBatch.length; i++) { + if (keepCurrentRow[i]) { + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + keepCurrentRow[i] = false; + } else if (paginationController.hasCurLimit()) { + builder.getTimeColumnBuilder().writeLong(timeBatch[i]); + builder.declarePosition(); + paginationController.consumeLimit(); + } else { + readEndIndex = i; + break; + } } } - } - // construct value columns - for (int i = 0; i < valueCount; i++) { - ValuePageReader pageReader = valuePageReaderList.get(i); - if (pageReader != null) { - pageReader.writeColumnBuilderWithNextBatch( - readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]); - } else { - for (int j = 0; j < readEndIndex; j++) { - if (keepCurrentRow[j]) { - builder.getColumnBuilder(i).appendNull(); + // construct value columns + for (int i = 0; i < valueCount; i++) { + ValuePageReader pageReader = valuePageReaderList.get(i); + if (pageReader != null) { + pageReader.writeColumnBuilderWithNextBatch( + readEndIndex, builder.getColumnBuilder(i), keepCurrentRow, isDeleted[i]); + } else { + for (int j = 0; j < readEndIndex; j++) { + if (keepCurrentRow[j]) { + builder.getColumnBuilder(i).appendNull(); + } } } } @@ -316,6 +355,6 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { @Override public void initTsBlockBuilder(List<TSDataType> dataTypes) { - builder = new TsBlockBuilder(dataTypes); + builder = new TsBlockBuilder((int) timePageReader.getStatistics().getCount(), dataTypes); } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java index 79e923c97f0..6018afd374e 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java @@ -329,6 +329,44 @@ public class ValuePageReader { } } + public void writeColumnBuilderWithNextBatch( + int readStartIndex, int readEndIndex, ColumnBuilder columnBuilder) { + if (valueBuffer == null) { + for (int i = readStartIndex; i < readEndIndex; i++) { + columnBuilder.appendNull(); + } + return; + } + for (int i = readStartIndex; i < readEndIndex; i++) { + if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { + columnBuilder.appendNull(); + continue; + } + switch (dataType) { + case BOOLEAN: + columnBuilder.writeBoolean(valueDecoder.readBoolean(valueBuffer)); + break; + case INT32: + columnBuilder.writeInt(valueDecoder.readInt(valueBuffer)); + break; + case INT64: + columnBuilder.writeLong(valueDecoder.readLong(valueBuffer)); + break; + case FLOAT: + columnBuilder.writeFloat(valueDecoder.readFloat(valueBuffer)); + break; + case DOUBLE: + columnBuilder.writeDouble(valueDecoder.readDouble(valueBuffer)); + break; + case TEXT: + columnBuilder.writeBinary(valueDecoder.readBinary(valueBuffer)); + break; + default: + throw new UnSupportedDataTypeException(String.valueOf(dataType)); + } + } + } + public Statistics getStatistics() { return pageHeader.getStatistics(); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java index 108fa0d0a5b..aa0f7971e47 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/PaginationController.java @@ -60,6 +60,14 @@ public class PaginationController { curOffset--; } + public long getCurOffset() { + return curOffset; + } + + public long getCurLimit() { + return curLimit; + } + public void consumeLimit() { if (hasLimit) { curLimit--;
