This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch fix-regex in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1bf41662bbd9543afdbe34de676889a666bc12e7 Author: Liu Xuxin <[email protected]> AuthorDate: Tue Oct 11 17:17:22 2022 +0800 fix dos attack --- .../resources/conf/iotdb-engine.properties | 4 ++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++++ .../iotdb/tsfile/common/conf/TSFileConfig.java | 10 +++++ .../iotdb/tsfile/read/filter/operator/Regexp.java | 48 +++++++++++++++++++++- 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 8991b9933a..f8267b9c38 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -741,6 +741,10 @@ timestamp_precision=ms # Datatype: long # slow_query_threshold=5000 +# max pattern access time in regex filter +# Datatype: int +# pattern_matching_threshold=1000000 + #################### ### MQTT Broker Configuration #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index d92bd997e9..f4517811da 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -792,6 +792,8 @@ public class IoTDBConfig { /** time cost(ms) threshold for slow query. Unit: millisecond */ private long slowQueryThreshold = 5000; + private int patternMatchingThreshold = 1000000; + /** * whether enable the rpc service. This parameter has no a corresponding field in the * iotdb-engine.properties @@ -2796,4 +2798,12 @@ public class IoTDBConfig { public void setChunkMetadataMemorySizeProportion(double chunkMetadataMemorySizeProportion) { this.chunkMetadataMemorySizeProportion = chunkMetadataMemorySizeProportion; } + + public int getPatternMatchingThreshold() { + return patternMatchingThreshold; + } + + public void setPatternMatchingThreshold(int patternMatchingThreshold) { + this.patternMatchingThreshold = patternMatchingThreshold; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7516081328..fadf77ccf0 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -928,6 +928,13 @@ public class IoTDBDescriptor { .setDfsClientFailoverProxyProvider( properties.getProperty( "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider())); + TSFileDescriptor.getInstance() + .getConfig() + .setPatternMatchingThreshold( + Integer.parseInt( + properties.getProperty( + "pattern_matching_threshold", + String.valueOf(conf.getPatternMatchingThreshold())))); TSFileDescriptor.getInstance() .getConfig() .setUseKerberos( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index df5df28446..0fd1bdcaa6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -146,6 +146,8 @@ public class TSFileConfig implements Serializable { private double bloomFilterErrorRate = 0.05; /** The amount of data iterate each time */ private int batchSize = 1000; + + private int patternMatchingThreshold = 1000000; /** customizedProperties, this should be empty by default. */ private Properties customizedProperties = new Properties(); @@ -422,4 +424,12 @@ public class TSFileConfig implements Serializable { public void setCustomizedProperties(Properties customizedProperties) { this.customizedProperties = customizedProperties; } + + public int getPatternMatchingThreshold() { + return patternMatchingThreshold; + } + + public void setPatternMatchingThreshold(int patternMatchingThreshold) { + this.patternMatchingThreshold = patternMatchingThreshold; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java index 646b51ba57..31e7260469 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.read.filter.operator; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId; @@ -66,7 +67,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { if (filterType != FilterType.VALUE_FILTER) { return false; } - return pattern.matcher(value.toString()).find(); + return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); } @Override @@ -124,4 +125,49 @@ public class Regexp<T extends Comparable<T>> implements Filter { public FilterSerializeId getSerializeId() { return FilterSerializeId.REGEXP; } + + private static class AccessCount { + private int count; + private final int accessThreshold = + TSFileDescriptor.getInstance().getConfig().getPatternMatchingThreshold(); + + public void check() throws IllegalStateException { + if (this.count++ > accessThreshold) { + throw new IllegalStateException("Pattern access threshold exceeded"); + } + } + } + + private static class MatcherInput implements CharSequence { + + private final CharSequence value; + + private final AccessCount access; + + public MatcherInput(CharSequence value, AccessCount access) { + this.value = value; + this.access = access; + } + + @Override + public char charAt(int index) { + this.access.check(); + return this.value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return new MatcherInput(this.value.subSequence(start, end), this.access); + } + + @Override + public int length() { + return this.value.length(); + } + + @Override + public String toString() { + return this.value.toString(); + } + } }
