JackieTien97 commented on code in PR #16246:
URL: https://github.com/apache/iotdb/pull/16246#discussion_r2306722293
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java:
##########
@@ -649,72 +655,114 @@ public void unlockQueryList() {
}
public TVListIterator iterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
- return new TVListIterator(deletionList, floatPrecision, encoding,
maxNumberOfPointsInPage);
+ return new TVListIterator(
+ scanOrder,
+ globalTimeFilter,
+ deletionList,
+ floatPrecision,
+ encoding,
+ maxNumberOfPointsInPage);
}
/* TVList Iterator */
- public class TVListIterator implements MemPointIterator {
+ public class TVListIterator extends MemPointIterator {
protected int index;
protected int rows;
protected boolean probeNext;
- protected List<TsBlock> tsBlocks;
+ protected final Filter globalTimeFilter;
private final List<TimeRange> deletionList;
- private final int[] deleteCursor = {0};
+ private final int[] deleteCursor;
private final int floatPrecision;
private final TSEncoding encoding;
// used by nextBatch during query
protected final int maxNumberOfPointsInPage;
public TVListIterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
+ super(scanOrder);
+ this.globalTimeFilter = globalTimeFilter;
this.deletionList = deletionList;
this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
this.encoding = encoding;
this.index = 0;
this.rows = rowCount;
this.probeNext = false;
- this.tsBlocks = new ArrayList<>();
this.maxNumberOfPointsInPage = maxNumberOfPointsInPage;
+ int cursor =
+ (deletionList == null || scanOrder.isAscending()) ? 0 :
(deletionList.size() - 1);
+ deleteCursor = new int[] {cursor};
}
protected void prepareNext() {
- // skip deleted rows
- while (index < rows
- && (isNullValue(getValueIndex(index))
- || isPointDeleted(getTime(index), deletionList, deleteCursor))) {
- index++;
- }
// skip duplicated timestamp
- while (index + 1 < rows && getTime(index + 1) == getTime(index)) {
- index++;
+ if (scanOrder.isAscending()) {
+ // skip deleted rows
+ skipDeletedRows();
+ while (index + 1 < rows
+ && getTime(getScanOrderIndex(index + 1)) ==
getTime(getScanOrderIndex(index))) {
+ index++;
+ }
+ } else {
+ while (index > 0
+ && (index <= rows - 1)
+ && getTime(getScanOrderIndex(index - 1)) ==
getTime(getScanOrderIndex(index))) {
+ index++;
+ }
+ skipDeletedRows();
}
probeNext = true;
}
+ protected void skipDeletedRows() {
+ while (index < rows) {
+ if (!isNullValue(getValueIndex(getScanOrderIndex(index)))) {
+ long time = getTime(getScanOrderIndex(index));
+ if (!isPointDeleted(time, deletionList, deleteCursor, scanOrder)
+ && isTimeSatisfied(time)) {
Review Comment:
It seems that this method doesn't only skip deleted rows, but also skip
time-unsatisfied rows?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java:
##########
@@ -747,73 +798,91 @@ public TsBlock nextBatch() {
switch (dataType) {
case BOOLEAN:
while (index < rows && builder.getPositionCount() <
maxNumberOfPointsInPage) {
- long time = getTime(index);
- if (!isNullValue(getValueIndex(index))
- && !isPointDeleted(time, deletionList, deleteCursor)
- && (index == rows - 1 || time != getTime(index + 1))) {
- builder.getTimeColumnBuilder().writeLong(time);
- builder.getColumnBuilder(0).writeBoolean(getBoolean(index));
- builder.declarePosition();
+ long time = getTime(getScanOrderIndex(index));
+ if (!isNullValue(getValueIndex(getScanOrderIndex(index)))
+ && !isPointDeleted(time, deletionList, deleteCursor, scanOrder)
+ && isTimeSatisfied(time)
+ && isLatestPoint(index, time)) {
+ boolean aBoolean = getBoolean(getScanOrderIndex(index));
+ if (pushDownFilter == null ||
pushDownFilter.satisfyBoolean(time, aBoolean)) {
+ builder.getTimeColumnBuilder().writeLong(time);
+ builder.getColumnBuilder(0).writeBoolean(aBoolean);
+ builder.declarePosition();
+ }
}
index++;
}
break;
case INT32:
case DATE:
while (index < rows && builder.getPositionCount() <
maxNumberOfPointsInPage) {
- long time = getTime(index);
- if (!isNullValue(getValueIndex(index))
- && !isPointDeleted(time, deletionList, deleteCursor)
- && (index == rows - 1 || time != getTime(index + 1))) {
- builder.getTimeColumnBuilder().writeLong(time);
- builder.getColumnBuilder(0).writeInt(getInt(index));
- builder.declarePosition();
+ long time = getTime(getScanOrderIndex(index));
+ if (!isNullValue(getValueIndex(getScanOrderIndex(index)))
+ && !isPointDeleted(time, deletionList, deleteCursor, scanOrder)
+ && isTimeSatisfied(time)
+ && isLatestPoint(index, time)) {
Review Comment:
change all the below
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java:
##########
@@ -649,72 +655,114 @@ public void unlockQueryList() {
}
public TVListIterator iterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
- return new TVListIterator(deletionList, floatPrecision, encoding,
maxNumberOfPointsInPage);
+ return new TVListIterator(
+ scanOrder,
+ globalTimeFilter,
+ deletionList,
+ floatPrecision,
+ encoding,
+ maxNumberOfPointsInPage);
}
/* TVList Iterator */
- public class TVListIterator implements MemPointIterator {
+ public class TVListIterator extends MemPointIterator {
protected int index;
protected int rows;
protected boolean probeNext;
- protected List<TsBlock> tsBlocks;
+ protected final Filter globalTimeFilter;
private final List<TimeRange> deletionList;
- private final int[] deleteCursor = {0};
+ private final int[] deleteCursor;
private final int floatPrecision;
private final TSEncoding encoding;
// used by nextBatch during query
protected final int maxNumberOfPointsInPage;
public TVListIterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
+ super(scanOrder);
+ this.globalTimeFilter = globalTimeFilter;
this.deletionList = deletionList;
this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
this.encoding = encoding;
this.index = 0;
this.rows = rowCount;
this.probeNext = false;
- this.tsBlocks = new ArrayList<>();
this.maxNumberOfPointsInPage = maxNumberOfPointsInPage;
+ int cursor =
+ (deletionList == null || scanOrder.isAscending()) ? 0 :
(deletionList.size() - 1);
+ deleteCursor = new int[] {cursor};
}
protected void prepareNext() {
- // skip deleted rows
- while (index < rows
- && (isNullValue(getValueIndex(index))
- || isPointDeleted(getTime(index), deletionList, deleteCursor))) {
- index++;
- }
// skip duplicated timestamp
Review Comment:
change this comments' line position
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java:
##########
@@ -1613,36 +1639,55 @@ protected void prepareNext() {
probeNext = true;
return;
}
- Arrays.fill(selectedIndices, index);
+ if (scanOrder.isAscending()) {
+ Arrays.fill(selectedIndices, index);
+ } else {
+ for (int i = 0; i < selectedIndices.length; i++) {
+ selectedIndices[i] = isNullValue(index, i) ? -1 : index;
+ }
Review Comment:
add some comments about else-branch, explain why we should differ in desc
order case
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java:
##########
@@ -747,73 +798,91 @@ public TsBlock nextBatch() {
switch (dataType) {
case BOOLEAN:
while (index < rows && builder.getPositionCount() <
maxNumberOfPointsInPage) {
- long time = getTime(index);
- if (!isNullValue(getValueIndex(index))
- && !isPointDeleted(time, deletionList, deleteCursor)
- && (index == rows - 1 || time != getTime(index + 1))) {
- builder.getTimeColumnBuilder().writeLong(time);
- builder.getColumnBuilder(0).writeBoolean(getBoolean(index));
- builder.declarePosition();
+ long time = getTime(getScanOrderIndex(index));
+ if (!isNullValue(getValueIndex(getScanOrderIndex(index)))
+ && !isPointDeleted(time, deletionList, deleteCursor, scanOrder)
+ && isTimeSatisfied(time)
+ && isLatestPoint(index, time)) {
+ boolean aBoolean = getBoolean(getScanOrderIndex(index));
+ if (pushDownFilter == null ||
pushDownFilter.satisfyBoolean(time, aBoolean)) {
+ builder.getTimeColumnBuilder().writeLong(time);
+ builder.getColumnBuilder(0).writeBoolean(aBoolean);
+ builder.declarePosition();
+ }
}
index++;
}
break;
case INT32:
case DATE:
while (index < rows && builder.getPositionCount() <
maxNumberOfPointsInPage) {
- long time = getTime(index);
- if (!isNullValue(getValueIndex(index))
- && !isPointDeleted(time, deletionList, deleteCursor)
- && (index == rows - 1 || time != getTime(index + 1))) {
- builder.getTimeColumnBuilder().writeLong(time);
- builder.getColumnBuilder(0).writeInt(getInt(index));
- builder.declarePosition();
+ long time = getTime(getScanOrderIndex(index));
+ if (!isNullValue(getValueIndex(getScanOrderIndex(index)))
+ && !isPointDeleted(time, deletionList, deleteCursor, scanOrder)
+ && isTimeSatisfied(time)
+ && isLatestPoint(index, time)) {
Review Comment:
```suggestion
&& isLatestPoint(index, time)
&& isTimeSatisfied(time)) {
```
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java:
##########
@@ -649,72 +655,114 @@ public void unlockQueryList() {
}
public TVListIterator iterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
- return new TVListIterator(deletionList, floatPrecision, encoding,
maxNumberOfPointsInPage);
+ return new TVListIterator(
+ scanOrder,
+ globalTimeFilter,
+ deletionList,
+ floatPrecision,
+ encoding,
+ maxNumberOfPointsInPage);
}
/* TVList Iterator */
- public class TVListIterator implements MemPointIterator {
+ public class TVListIterator extends MemPointIterator {
protected int index;
protected int rows;
protected boolean probeNext;
- protected List<TsBlock> tsBlocks;
+ protected final Filter globalTimeFilter;
private final List<TimeRange> deletionList;
- private final int[] deleteCursor = {0};
+ private final int[] deleteCursor;
private final int floatPrecision;
private final TSEncoding encoding;
// used by nextBatch during query
protected final int maxNumberOfPointsInPage;
public TVListIterator(
+ Ordering scanOrder,
+ Filter globalTimeFilter,
List<TimeRange> deletionList,
Integer floatPrecision,
TSEncoding encoding,
int maxNumberOfPointsInPage) {
+ super(scanOrder);
+ this.globalTimeFilter = globalTimeFilter;
this.deletionList = deletionList;
this.floatPrecision = floatPrecision != null ? floatPrecision : 0;
this.encoding = encoding;
this.index = 0;
this.rows = rowCount;
this.probeNext = false;
- this.tsBlocks = new ArrayList<>();
this.maxNumberOfPointsInPage = maxNumberOfPointsInPage;
+ int cursor =
+ (deletionList == null || scanOrder.isAscending()) ? 0 :
(deletionList.size() - 1);
+ deleteCursor = new int[] {cursor};
}
protected void prepareNext() {
- // skip deleted rows
- while (index < rows
- && (isNullValue(getValueIndex(index))
- || isPointDeleted(getTime(index), deletionList, deleteCursor))) {
- index++;
- }
// skip duplicated timestamp
- while (index + 1 < rows && getTime(index + 1) == getTime(index)) {
- index++;
+ if (scanOrder.isAscending()) {
+ // skip deleted rows
+ skipDeletedRows();
+ while (index + 1 < rows
+ && getTime(getScanOrderIndex(index + 1)) ==
getTime(getScanOrderIndex(index))) {
+ index++;
+ }
+ } else {
+ while (index > 0
Review Comment:
add comments for else-branch, also also explain why we need to skip
duplicated firstly for desc scan order which is opposite to asc scan order
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java:
##########
@@ -1613,36 +1639,55 @@ protected void prepareNext() {
probeNext = true;
return;
}
- Arrays.fill(selectedIndices, index);
+ if (scanOrder.isAscending()) {
+ Arrays.fill(selectedIndices, index);
+ } else {
+ for (int i = 0; i < selectedIndices.length; i++) {
+ selectedIndices[i] = isNullValue(index, i) ? -1 : index;
+ }
+ }
findValidRow = true;
// handle duplicated timestamp
- while (index + 1 < rows && getTime(index + 1) == getTime(index)) {
+ while (index + 1 < rows
+ && getTime(getScanOrderIndex(index + 1)) ==
getTime(getScanOrderIndex(index))) {
Review Comment:
why we don't need to handle the order of duplicated timestamp and delete
judgement in different scan order cases, just like you do in TVList
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java:
##########
@@ -1807,15 +1862,24 @@ public TsBlock nextBatch() {
for (int sortedRowIndex = startIndex; sortedRowIndex < index;
sortedRowIndex++) {
// skip empty row
if ((allValueColDeletedMap != null
- &&
allValueColDeletedMap.isMarked(getValueIndex(sortedRowIndex)))
- || (isTimeDeleted(sortedRowIndex))) {
+ && allValueColDeletedMap.isMarked(
+ getValueIndex(getScanOrderIndex(sortedRowIndex))))
+ || (isTimeDeleted(getScanOrderIndex(sortedRowIndex))
+ ||
!isTimeSatisfied(getTime(getScanOrderIndex(sortedRowIndex))))) {
continue;
}
// skip time duplicated or totally deleted rows
if (Objects.nonNull(timeInvalidInfo)) {
- if (!outer.isNullValue(getValueIndex(sortedRowIndex),
validColumnIndex)) {
- lastValidPointIndexForTimeDupCheck.left =
getTime(sortedRowIndex);
- lastValidPointIndexForTimeDupCheck.right =
getValueIndex(sortedRowIndex);
+ if (!outer.isNullValue(
+ getValueIndex(getScanOrderIndex(sortedRowIndex)),
validColumnIndex)) {
+ lastValidPointIndexForTimeDupCheck.left =
getTime(getScanOrderIndex(sortedRowIndex));
+ if (scanOrder.isAscending()) {
+ lastValidPointIndexForTimeDupCheck.right =
+ getValueIndex(getScanOrderIndex(sortedRowIndex));
+ } else if (lastValidPointIndexForTimeDupCheck.right == null) {
+ lastValidPointIndexForTimeDupCheck.right =
+ getValueIndex(getScanOrderIndex(sortedRowIndex));
+ }
Review Comment:
add some comments
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java:
##########
@@ -1839,61 +1903,28 @@ public TsBlock nextBatch() {
// write(T:3,V:null)
int originRowIndex;
if (Objects.nonNull(lastValidPointIndexForTimeDupCheck)
- && (getTime(sortedRowIndex) ==
lastValidPointIndexForTimeDupCheck.left)) {
+ && (getTime(getScanOrderIndex(sortedRowIndex))
+ == lastValidPointIndexForTimeDupCheck.left)
+ && Objects.nonNull(lastValidPointIndexForTimeDupCheck.right)) {
originRowIndex = lastValidPointIndexForTimeDupCheck.right;
+ lastValidPointIndexForTimeDupCheck.right = null;
} else {
- originRowIndex = getValueIndex(sortedRowIndex);
+ originRowIndex = getValueIndex(getScanOrderIndex(sortedRowIndex));
Review Comment:
add some comments
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
