This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/likeDebug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8486e3b663e41da7531dfa22a73a66e75481a398 Author: Minghui Liu <[email protected]> AuthorDate: Thu Oct 27 14:32:04 2022 +0800 Implement `NOT REGEXP` filter --- .../iotdb/db/qp/logical/crud/RegexpOperator.java | 40 ++++++++++++++++------ .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 2 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +- .../iotdb/tsfile/read/filter/ValueFilter.java | 19 +++++----- .../iotdb/tsfile/read/filter/operator/Like.java | 9 +++++ .../iotdb/tsfile/read/filter/operator/Regexp.java | 20 +++++++---- .../tsfile/read/filter/FilterSerializeTest.java | 2 +- 7 files changed, 66 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/RegexpOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/RegexpOperator.java index 4551f09f2a..a411de9a8e 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/RegexpOperator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/RegexpOperator.java @@ -37,12 +37,14 @@ import static org.apache.iotdb.tsfile.file.metadata.enums.TSDataType.TEXT; public class RegexpOperator extends FunctionOperator { + private boolean not; protected String value; - public RegexpOperator(FilterType filterType, PartialPath path, String value) { + public RegexpOperator(FilterType filterType, PartialPath path, String value, boolean not) { super(filterType); this.singlePath = path; this.value = value; + this.not = not; isLeaf = true; isSingle = true; } @@ -67,19 +69,20 @@ public class RegexpOperator extends FunctionOperator { singlePath, (value.startsWith("'") && value.endsWith("'")) ? value.substring(1, value.length() - 1) - : value); + : value, + not); } return new Pair<>(ret, singlePath.getFullPath()); } private static class Regexp { public static <T extends Comparable<T>> IUnaryExpression getUnaryExpression( - PartialPath path, String value) { - return new SingleSeriesExpression(path, ValueFilter.regexp(value)); + PartialPath path, String value, boolean not) { + return new SingleSeriesExpression(path, ValueFilter.regexp(value, not)); } - public <T extends Comparable<T>> Filter getValueFilter(String value) { - return ValueFilter.regexp(value); + public <T extends Comparable<T>> Filter getValueFilter(String value, boolean not) { + return ValueFilter.regexp(value, not); } } @@ -89,13 +92,13 @@ public class RegexpOperator extends FunctionOperator { for (int i = 0; i < spaceNum; i++) { sc.addTail(" "); } - sc.addTail(singlePath.getFullPath(), value, ", single\n"); + sc.addTail(singlePath.getFullPath(), not, value, ", single\n"); return sc.toString(); } @Override public RegexpOperator copy() { - RegexpOperator ret = new RegexpOperator(this.filterType, singlePath.clone(), value); + RegexpOperator ret = new RegexpOperator(this.filterType, singlePath.clone(), value, not); ret.isLeaf = isLeaf; ret.isSingle = isSingle; ret.pathSet = pathSet; @@ -114,20 +117,35 @@ public class RegexpOperator extends FunctionOperator { return false; } RegexpOperator that = (RegexpOperator) o; - return Objects.equals(value, that.value); + return Objects.equals(value, that.value) && not == that.not; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), singlePath, value); + return Objects.hash(super.hashCode(), singlePath, value, not); } @Override public String toString() { - return "[" + singlePath.getFullPath() + value + "]"; + return "[" + + singlePath.getFullPath() + + (not ? " NOT " : " ") + + getFilterSymbol() + + " " + + value + + "]"; + } + + @Override + public void reverseFunc() { + not = !not; } public String getValue() { return value; } + + public boolean isNot() { + return not; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java index 7f2f969d2e..67ffe2efe3 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java @@ -2620,7 +2620,7 @@ public class IoTDBSqlVisitor extends IoTDBSqlParserBaseVisitor<Operator> { throw new SQLParserException("Path is null, please check the sql."); } return ctx.REGEXP() != null - ? new RegexpOperator(FilterType.REGEXP, path, ctx.STRING_LITERAL().getText()) + ? new RegexpOperator(FilterType.REGEXP, path, ctx.STRING_LITERAL().getText(), false) : new LikeOperator(FilterType.LIKE, path, ctx.STRING_LITERAL().getText(), false); } else { if (ctx.TIME() != null || ctx.TIMESTAMP() != null) { diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java index a3ffabe5d5..4a23f77f6c 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java @@ -350,7 +350,8 @@ public class ConcatPathOptimizer implements ILogicalOptimizer { new RegexpOperator( operator.getFilterType(), noStarPaths.get(i), - ((RegexpOperator) operator).getValue())); + ((RegexpOperator) operator).getValue(), + ((RegexpOperator) operator).isNot())); } else { currentNode.addChildOperator( new BasicFunctionOperator( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java index fd06538cb7..c6bc55f34e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java @@ -70,8 +70,8 @@ public class ValueFilter { return new ValueNotEq(value); } - public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value) { - return new ValueRegexp(value); + public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value, boolean not) { + return new ValueRegexp(value, not); } public static <T extends Comparable<T>> ValueLike<T> like(String value, boolean not) { @@ -246,8 +246,8 @@ public class ValueFilter { public static class ValueRegexp<T extends Comparable<T>> extends Regexp<T> { - private ValueRegexp(String value) { - super(value, FilterType.VALUE_FILTER); + private ValueRegexp(String value, boolean not) { + super(value, FilterType.VALUE_FILTER, not); } } @@ -255,14 +255,17 @@ public class ValueFilter { private final int index; - private VectorValueRegexp(String value, int index) { - super(value); + private VectorValueRegexp(String value, int index, boolean not) { + super(value, not); this.index = index; } public boolean satisfy(long time, TsPrimitiveType[] values) { - Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); - return this.value.equals(v); + if (filterType != FilterType.VALUE_FILTER) { + return false; + } + Object value = values[index].getValue(); + return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find() != not; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java index c3230e586b..7b7023d39a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java @@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Objects; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -134,6 +135,14 @@ public class Like<T extends Comparable<T>> implements Filter { not = ReadWriteIOUtils.readBool(buffer); } + @Override + public boolean equals(Object o) { + return o instanceof Like + && Objects.equals(((Like<?>) o).value, value) + && ((Like<?>) o).filterType == filterType + && ((Like<?>) o).not == not; + } + @Override public String toString() { return filterType + (not ? " not like " : " like ") + value; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java index 31e7260469..2ffed8a253 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java @@ -45,11 +45,14 @@ public class Regexp<T extends Comparable<T>> implements Filter { protected Pattern pattern; + protected boolean not; + public Regexp() {} - public Regexp(String value, FilterType filterType) { + public Regexp(String value, FilterType filterType, boolean not) { this.value = value; this.filterType = filterType; + this.not = not; try { this.pattern = Pattern.compile(this.value); } catch (PatternSyntaxException e) { @@ -67,7 +70,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { if (filterType != FilterType.VALUE_FILTER) { return false; } - return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); + return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find() != not; } @Override @@ -82,7 +85,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { @Override public Filter copy() { - return new Regexp(value, filterType); + return new Regexp(value, filterType, not); } @Override @@ -91,6 +94,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { outputStream.write(getSerializeId().ordinal()); outputStream.write(filterType.ordinal()); ReadWriteIOUtils.write(value, outputStream); + ReadWriteIOUtils.write(not, outputStream); } catch (IOException ex) { throw new IllegalArgumentException("Failed to serialize outputStream of type:", ex); } @@ -107,18 +111,20 @@ public class Regexp<T extends Comparable<T>> implements Filter { throw new PatternSyntaxException("Regular expression error", value, e.getIndex()); } } + not = ReadWriteIOUtils.readBool(buffer); } @Override public String toString() { - return filterType + " is " + value; + return filterType + (not ? " not regexp " : " regexp ") + value; } @Override public boolean equals(Object o) { return o instanceof Regexp && Objects.equals(((Regexp<?>) o).value, value) - && ((Regexp<?>) o).filterType == filterType; + && ((Regexp<?>) o).filterType == filterType + && ((Regexp<?>) o).not == not; } @Override @@ -126,7 +132,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { return FilterSerializeId.REGEXP; } - private static class AccessCount { + public static class AccessCount { private int count; private final int accessThreshold = TSFileDescriptor.getInstance().getConfig().getPatternMatchingThreshold(); @@ -138,7 +144,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { } } - private static class MatcherInput implements CharSequence { + public static class MatcherInput implements CharSequence { private final CharSequence value; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java index 779e32d404..fc8e63a16f 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java @@ -48,7 +48,7 @@ public class FilterSerializeTest { ValueFilter.notEq(false), ValueFilter.in(new HashSet<>(Arrays.asList("a", "b")), false), ValueFilter.in(new HashSet<>(Arrays.asList("c", "d")), true), - ValueFilter.regexp("s.*"), + ValueFilter.regexp("s.*", false), }; for (Filter filter : filters) { validateSerialization(filter);
