This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 772838c [IOTDB-2023] Fix serializing and deserializing bugs of
Filters (#4414)
772838c is described below
commit 772838cfd2b5e784ccb72adcac7385f98da50344
Author: BaiJian <[email protected]>
AuthorDate: Thu Nov 18 10:16:12 2021 +0800
[IOTDB-2023] Fix serializing and deserializing bugs of Filters (#4414)
---
.../tsfile/read/filter/GroupByMonthFilter.java | 114 +++++++++++++++++----
.../tsfile/read/filter/factory/FilterFactory.java | 8 ++
.../read/filter/factory/FilterSerializeId.java | 1 +
.../iotdb/tsfile/read/filter/operator/In.java | 15 ++-
.../tsfile/read/filter/FilterSerializeTest.java | 25 ++++-
5 files changed, 137 insertions(+), 26 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
index 213d520..b36ba41 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
@@ -19,7 +19,12 @@
package org.apache.iotdb.tsfile.read.filter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Calendar;
import java.util.Objects;
import java.util.TimeZone;
@@ -30,14 +35,23 @@ import java.util.TimeZone;
*/
public class GroupByMonthFilter extends GroupByFilter {
- private final boolean isSlidingStepByMonth;
- private final boolean isIntervalByMonth;
private int slidingStepsInMo;
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
- private final long initialStartTime;
+ private long initialStartTime;
+
+ // These fields will be serialized to remote nodes, as other fields may be
updated during process
+ private TimeZone timeZone;
+ private boolean isSlidingStepByMonth;
+ private boolean isIntervalByMonth;
+ private long originalSlidingStep;
+ private long originalInterval;
+ private long originalStartTime;
+ private long originalEndTime;
+
+ public GroupByMonthFilter() {}
public GroupByMonthFilter(
long interval,
@@ -48,19 +62,11 @@ public class GroupByMonthFilter extends GroupByFilter {
boolean isIntervalByMonth,
TimeZone timeZone) {
super(interval, slidingStep, startTime, endTime);
- initialStartTime = startTime;
- calendar.setTimeZone(timeZone);
- calendar.setTimeInMillis(startTime);
- this.isIntervalByMonth = isIntervalByMonth;
- this.isSlidingStepByMonth = isSlidingStepByMonth;
- if (isIntervalByMonth) {
- // TODO: 1mo1d
- intervalInMo = (int) (interval / MS_TO_MONTH);
- }
- if (isSlidingStepByMonth) {
- slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
- }
- getNthTimeInterval(0);
+ this.originalInterval = interval;
+ this.originalSlidingStep = slidingStep;
+ this.originalStartTime = startTime;
+ this.originalEndTime = endTime;
+ initMonthGroupByParameters(isSlidingStepByMonth, isIntervalByMonth,
timeZone);
}
public GroupByMonthFilter(GroupByMonthFilter filter) {
@@ -70,9 +76,14 @@ public class GroupByMonthFilter extends GroupByFilter {
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
initialStartTime = filter.initialStartTime;
+ originalStartTime = filter.originalStartTime;
+ originalEndTime = filter.originalEndTime;
+ originalSlidingStep = filter.originalSlidingStep;
+ originalInterval = filter.originalInterval;
calendar = Calendar.getInstance();
calendar.setTimeZone(filter.calendar.getTimeZone());
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
+ timeZone = filter.timeZone;
}
// TODO: time descending order
@@ -133,6 +144,40 @@ public class GroupByMonthFilter extends GroupByFilter {
}
}
+ @Override
+ public void serialize(DataOutputStream outputStream) {
+ try {
+ outputStream.write(getSerializeId().ordinal());
+ ReadWriteIOUtils.write(originalInterval, outputStream);
+ ReadWriteIOUtils.write(originalSlidingStep, outputStream);
+ ReadWriteIOUtils.write(originalStartTime, outputStream);
+ ReadWriteIOUtils.write(originalEndTime, outputStream);
+ ReadWriteIOUtils.write(isSlidingStepByMonth, outputStream);
+ ReadWriteIOUtils.write(isIntervalByMonth, outputStream);
+ ReadWriteIOUtils.write(timeZone.getID(), outputStream);
+ } catch (IOException ignored) {
+ // ignored
+ }
+ }
+
+ @Override
+ public void deserialize(ByteBuffer buffer) {
+ originalInterval = ReadWriteIOUtils.readLong(buffer);
+ originalSlidingStep = ReadWriteIOUtils.readLong(buffer);
+ originalStartTime = ReadWriteIOUtils.readLong(buffer);
+ originalEndTime = ReadWriteIOUtils.readLong(buffer);
+ isSlidingStepByMonth = ReadWriteIOUtils.readBool(buffer);
+ isIntervalByMonth = ReadWriteIOUtils.readBool(buffer);
+ timeZone = TimeZone.getTimeZone(ReadWriteIOUtils.readString(buffer));
+
+ interval = originalInterval;
+ slidingStep = originalSlidingStep;
+ startTime = originalStartTime;
+ endTime = originalEndTime;
+
+ initMonthGroupByParameters(isSlidingStepByMonth, isIntervalByMonth,
timeZone);
+ }
+
private boolean isContainedByCurrentInterval(long startTime, long endTime) {
if (startTime < this.startTime || endTime > this.endTime) {
return false;
@@ -147,12 +192,16 @@ public class GroupByMonthFilter extends GroupByFilter {
return false;
}
GroupByMonthFilter other = (GroupByMonthFilter) obj;
- return this.interval == other.interval
- && this.slidingStep == other.slidingStep
- && this.startTime == other.startTime
- && this.endTime == other.endTime
+ return this.originalInterval == other.originalInterval
+ && this.originalSlidingStep == other.originalSlidingStep
+ && this.originalStartTime == other.originalStartTime
+ && this.originalEndTime == other.originalEndTime
&& this.isSlidingStepByMonth == other.isSlidingStepByMonth
- && this.isIntervalByMonth == other.isIntervalByMonth;
+ && this.isIntervalByMonth == other.isIntervalByMonth
+ && this.timeZone.equals(other.timeZone)
+ && this.initialStartTime == other.initialStartTime
+ && this.intervalInMo == other.intervalInMo
+ && this.slidingStepsInMo == other.slidingStepsInMo;
}
@Override
@@ -161,6 +210,24 @@ public class GroupByMonthFilter extends GroupByFilter {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth,
isIntervalByMonth);
}
+ private void initMonthGroupByParameters(
+ boolean isSlidingStepByMonth, boolean isIntervalByMonth, TimeZone
timeZone) {
+ initialStartTime = startTime;
+ calendar.setTimeZone(timeZone);
+ calendar.setTimeInMillis(startTime);
+ this.timeZone = timeZone;
+ this.isIntervalByMonth = isIntervalByMonth;
+ this.isSlidingStepByMonth = isSlidingStepByMonth;
+ if (isIntervalByMonth) {
+ // TODO: 1mo1d
+ intervalInMo = (int) (interval / MS_TO_MONTH);
+ }
+ if (isSlidingStepByMonth) {
+ slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
+ }
+ getNthTimeInterval(0);
+ }
+
/** Get the interval that @param time belongs to. */
private long getTimePointPosition(long time) {
long count;
@@ -210,4 +277,9 @@ public class GroupByMonthFilter extends GroupByFilter {
this.slidingStep = calendar.getTimeInMillis() - startTime;
}
}
+
+ @Override
+ public FilterSerializeId getSerializeId() {
+ return FilterSerializeId.GROUP_BY_MONTH;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
index 61f0386..97f12e2 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
@@ -19,11 +19,13 @@
package org.apache.iotdb.tsfile.read.filter.factory;
import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
import org.apache.iotdb.tsfile.read.filter.operator.Eq;
import org.apache.iotdb.tsfile.read.filter.operator.Gt;
import org.apache.iotdb.tsfile.read.filter.operator.GtEq;
+import org.apache.iotdb.tsfile.read.filter.operator.In;
import org.apache.iotdb.tsfile.read.filter.operator.Lt;
import org.apache.iotdb.tsfile.read.filter.operator.LtEq;
import org.apache.iotdb.tsfile.read.filter.operator.NotEq;
@@ -78,9 +80,15 @@ public class FilterFactory {
case LTEQ:
filter = new LtEq<>();
break;
+ case IN:
+ filter = new In<>();
+ break;
case GROUP_BY:
filter = new GroupByFilter();
break;
+ case GROUP_BY_MONTH:
+ filter = new GroupByMonthFilter();
+ break;
default:
throw new UnsupportedOperationException("Unknown filter type " + id);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterSerializeId.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterSerializeId.java
index ec7f348..85b0a4f 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterSerializeId.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterSerializeId.java
@@ -23,6 +23,7 @@ public enum FilterSerializeId {
AND,
EQ,
GROUP_BY,
+ GROUP_BY_MONTH,
GT,
GTEQ,
LT,
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
index c76154a..60e0747 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
@@ -88,7 +88,7 @@ public class In<T extends Comparable<T>> implements Filter {
outputStream.write(getSerializeId().ordinal());
outputStream.write(filterType.ordinal());
ReadWriteIOUtils.write(not, outputStream);
- outputStream.write(values.size());
+ ReadWriteIOUtils.write(values.size(), outputStream);
for (T value : values) {
ReadWriteIOUtils.writeObject(value, outputStream);
}
@@ -101,13 +101,22 @@ public class In<T extends Comparable<T>> implements
Filter {
public void deserialize(ByteBuffer buffer) {
filterType = FilterType.values()[buffer.get()];
not = ReadWriteIOUtils.readBool(buffer);
- values = new HashSet<>();
- for (int i = 0; i < buffer.get(); i++) {
+ int size = ReadWriteIOUtils.readInt(buffer);
+ values = new HashSet<>(size);
+ for (int i = 0; i < size; i++) {
values.add((T) ReadWriteIOUtils.readObject(buffer));
}
}
@Override
+ public boolean equals(Object o) {
+ return o instanceof In
+ && ((In<?>) o).filterType == filterType
+ && ((In<?>) o).values.equals(values)
+ && ((In<?>) o).not == not;
+ }
+
+ @Override
public String toString() {
List<T> valueList = new ArrayList<>(values);
Collections.sort(valueList);
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
index 4e15225..2dd8a48 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
@@ -26,6 +26,9 @@ import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.TimeZone;
import static org.junit.Assert.assertEquals;
@@ -41,7 +44,10 @@ public class FilterSerializeTest {
ValueFilter.lt(0.1),
ValueFilter.ltEq(0.01f),
ValueFilter.not(ValueFilter.eq(true)),
- ValueFilter.notEq(false)
+ ValueFilter.notEq(false),
+ ValueFilter.notEq(false),
+ ValueFilter.in(new HashSet<>(Arrays.asList("a", "b")), false),
+ ValueFilter.in(new HashSet<>(Arrays.asList("c", "d")), true),
};
for (Filter filter : filters) {
validateSerialization(filter);
@@ -58,7 +64,10 @@ public class FilterSerializeTest {
TimeFilter.lt(4),
TimeFilter.ltEq(5),
TimeFilter.not(ValueFilter.eq(6)),
- TimeFilter.notEq(7)
+ TimeFilter.notEq(7),
+ TimeFilter.notEq(7),
+ TimeFilter.in(new HashSet<>(Arrays.asList(1L, 2L)), false),
+ TimeFilter.in(new HashSet<>(Arrays.asList(3L, 4L)), true),
};
for (Filter filter : filters) {
validateSerialization(filter);
@@ -88,6 +97,18 @@ public class FilterSerializeTest {
}
}
+ @Test
+ public void testGroupByMonthFilter() {
+ Filter[] filters =
+ new Filter[] {
+ new GroupByMonthFilter(1, 2, 3, 4, true, false,
TimeZone.getTimeZone("Asia/Shanghai")),
+ new GroupByMonthFilter(4, 3, 2, 1, false, true,
TimeZone.getTimeZone("Atlantic/Faeroe")),
+ };
+ for (Filter filter : filters) {
+ validateSerialization(filter);
+ }
+ }
+
private void validateSerialization(Filter filter) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(outputStream);
