Manybubbles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/215913
Change subject: Make regexes easier to use from other plugins
......................................................................
Make regexes easier to use from other plugins
1. SourceRegexFilter now implementes hashCode and equals so you can compare.
Most Lucene queries do this and it makes testing simpler.
2. Factor out many settings from the builder and parser so they can be built
and parsed independently of the plugin. This is useful for including them
in other plugins.
Change-Id: I9e93a1a6c0e7e4dab9e9ca46d23ec24b325a0887
---
M src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilter.java
M src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterBuilder.java
M src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterParser.java
M
src/main/java/org/wikimedia/search/extra/regex/UnableToAccelerateRegexException.java
M src/main/java/org/wikimedia/search/extra/regex/ngram/NGramExtractor.java
M src/main/java/org/wikimedia/search/extra/util/FieldValues.java
6 files changed, 424 insertions(+), 130 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/search/extra
refs/changes/13/215913/1
diff --git
a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilter.java
b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilter.java
index 7cdd399..7b2e483 100644
--- a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilter.java
+++ b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilter.java
@@ -24,39 +24,22 @@
public class SourceRegexFilter extends Filter {
private final String fieldPath;
+ private final String ngramFieldPath;
private final String regex;
private final FieldValues.Loader loader;
- private final String ngramFieldPath;
+ private final Settings settings;
private final int gramSize;
- private final int maxExpand;
- private final int maxStatesTraced;
- private final int maxDeterminizedStates;
- private final int maxNgramsExtracted;
- private final int maxInspect;
- private final boolean caseSensitive;
- private final Locale locale;
- private final boolean rejectUnaccelerated;
private int inspected = 0;
private Filter prefilter;
private XCharacterRunAutomaton charRun;
-
- public SourceRegexFilter(String fieldPath, FieldValues.Loader loader,
String regex, String ngramFieldPath, int gramSize, int maxExpand,
- int maxStatesTraced, int maxDeterminizedStates, int
maxNgramsExtracted, int maxInspect, boolean caseSensitive, Locale locale,
- boolean rejectUnaccelerated) {
+ public SourceRegexFilter(String fieldPath, String ngramFieldPath, String
regex, FieldValues.Loader loader, Settings settings, int gramSize) {
this.fieldPath = fieldPath;
- this.loader = loader;
- this.regex = regex;
this.ngramFieldPath = ngramFieldPath;
+ this.regex = regex;
+ this.loader = loader;
+ this.settings = settings;
this.gramSize = gramSize;
- this.maxExpand = maxExpand;
- this.maxStatesTraced = maxStatesTraced;
- this.maxDeterminizedStates = maxDeterminizedStates;
- this.maxNgramsExtracted = maxNgramsExtracted;
- this.maxInspect = maxInspect;
- this.caseSensitive = caseSensitive;
- this.locale = locale;
- this.rejectUnaccelerated = rejectUnaccelerated;
}
@Override
@@ -72,15 +55,20 @@
if (ngramFieldPath == null) {
// Don't bother expanding the regex if there isn't a field to check
// it against. Its unlikely to resolve to all false anyway.
+ if (settings.getRejectUnaccelerated()) {
+ throw new UnableToAccelerateRegexException(regex, gramSize,
ngramFieldPath);
+ }
return new AllDocIdSet(context.reader().maxDoc());
}
if (prefilter == null) {
try {
// The accelerating filter is always assumed to be case
insensitive/always lowercased
- XAutomaton automaton = new XRegExp(regex.toLowerCase(locale),
XRegExp.ALL ^ XRegExp.AUTOMATON).toAutomaton(maxDeterminizedStates);
- Expression<String> expression = new NGramExtractor(gramSize,
maxExpand, maxStatesTraced, maxNgramsExtracted).extract(automaton).simplify();
+ XAutomaton automaton = new
XRegExp(regex.toLowerCase(settings.getLocale()),
+ XRegExp.ALL ^
XRegExp.AUTOMATON).toAutomaton(settings.getMaxDeterminizedStates());
+ Expression<String> expression = new NGramExtractor(gramSize,
settings.getMaxExpand(),
+ settings.getMaxStatesTraced(),
settings.getMaxNgramsExtracted()).extract(automaton).simplify();
if (expression.alwaysTrue()) {
- if (rejectUnaccelerated) {
+ if (settings.getRejectUnaccelerated()) {
throw new UnableToAccelerateRegexException(regex,
gramSize, ngramFieldPath);
}
prefilter = Queries.MATCH_ALL_FILTER;
@@ -92,7 +80,7 @@
} catch (AutomatonTooComplexException e) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
"Regex /%s/ too complex for maxStatesTraced setting
[%s]. Use a simpler regex or raise maxStatesTraced.", regex,
- maxStatesTraced), e);
+ settings.getMaxStatesTraced()), e);
}
}
return prefilter.getDocIdSet(context, acceptDocs);
@@ -112,23 +100,24 @@
@Override
protected boolean match(int docid) {
- if (inspected >= maxInspect) {
+ if (inspected >= settings.getMaxInspect()) {
// TODO hook into the generic timeout mechanism when it is
ready
return false;
}
inspected++;
if (charRun == null) {
String regexString = regex;
- if (!caseSensitive) {
- regexString = regexString.toLowerCase(locale);
+ if (!settings.getCaseSensitive()) {
+ regexString =
regexString.toLowerCase(settings.getLocale());
}
- XAutomaton automaton = new XRegExp(".*" + regexString + ".*",
XRegExp.ALL ^ XRegExp.AUTOMATON).toAutomaton(maxDeterminizedStates);
+ XAutomaton automaton = new XRegExp(".*" + regexString + ".*",
XRegExp.ALL ^ XRegExp.AUTOMATON).toAutomaton(settings
+ .getMaxDeterminizedStates());
charRun = new XCharacterRunAutomaton(automaton);
}
List<String> values = load(docid);
for (String value : values) {
- if (!caseSensitive) {
- value = value.toLowerCase(locale);
+ if (!settings.getCaseSensitive()) {
+ value = value.toLowerCase(settings.getLocale());
}
if (charRun.run(value)) {
return true;
@@ -145,4 +134,187 @@
}
}
}
+
+ @Override
+ public String toString() {
+ StringBuilder b = new StringBuilder();
+ b.append(fieldPath).append(":/").append(regex).append('/');
+ if (ngramFieldPath != null) {
+ b.append('~').append(ngramFieldPath);
+ }
+ return b.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((fieldPath == null) ? 0 :
fieldPath.hashCode());
+ result = prime * result + gramSize;
+ result = prime * result + ((loader == null) ? 0 : loader.hashCode());
+ result = prime * result + ((ngramFieldPath == null) ? 0 :
ngramFieldPath.hashCode());
+ result = prime * result + ((regex == null) ? 0 : regex.hashCode());
+ result = prime * result + ((settings == null) ? 0 :
settings.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SourceRegexFilter other = (SourceRegexFilter) obj;
+ if (fieldPath == null) {
+ if (other.fieldPath != null)
+ return false;
+ } else if (!fieldPath.equals(other.fieldPath))
+ return false;
+ if (gramSize != other.gramSize)
+ return false;
+ if (loader == null) {
+ if (other.loader != null)
+ return false;
+ } else if (!loader.equals(other.loader))
+ return false;
+ if (ngramFieldPath == null) {
+ if (other.ngramFieldPath != null)
+ return false;
+ } else if (!ngramFieldPath.equals(other.ngramFieldPath))
+ return false;
+ if (regex == null) {
+ if (other.regex != null)
+ return false;
+ } else if (!regex.equals(other.regex))
+ return false;
+ if (settings == null) {
+ if (other.settings != null)
+ return false;
+ } else if (!settings.equals(other.settings))
+ return false;
+ return true;
+ }
+
+ public static class Settings {
+ private int maxExpand = 4;
+ private int maxStatesTraced = 10000;
+ private int maxDeterminizedStates = 20000;
+ private int maxNgramsExtracted = 100;
+ private int maxInspect = Integer.MAX_VALUE;
+ private boolean caseSensitive = false;
+ private Locale locale = Locale.ROOT;
+ private boolean rejectUnaccelerated = false;
+
+ public int getMaxExpand() {
+ return maxExpand;
+ }
+
+ public void setMaxExpand(int maxExpand) {
+ this.maxExpand = maxExpand;
+ }
+
+ public int getMaxStatesTraced() {
+ return maxStatesTraced;
+ }
+
+ public void setMaxStatesTraced(int maxStatesTraced) {
+ this.maxStatesTraced = maxStatesTraced;
+ }
+
+ public int getMaxDeterminizedStates() {
+ return maxDeterminizedStates;
+ }
+
+ public void setMaxDeterminizedStates(int maxDeterminizedStates) {
+ this.maxDeterminizedStates = maxDeterminizedStates;
+ }
+
+ public int getMaxNgramsExtracted() {
+ return maxNgramsExtracted;
+ }
+
+ public void setMaxNgramsExtracted(int maxNgramsExtracted) {
+ this.maxNgramsExtracted = maxNgramsExtracted;
+ }
+
+ public int getMaxInspect() {
+ return maxInspect;
+ }
+
+ public void setMaxInspect(int maxInspect) {
+ this.maxInspect = maxInspect;
+ }
+
+ public boolean getCaseSensitive() {
+ return caseSensitive;
+ }
+
+ public void setCaseSensitive(boolean caseSensitive) {
+ this.caseSensitive = caseSensitive;
+ }
+
+ public Locale getLocale() {
+ return locale;
+ }
+
+ public void setLocale(Locale locale) {
+ this.locale = locale;
+ }
+
+ public boolean getRejectUnaccelerated() {
+ return rejectUnaccelerated;
+ }
+
+ public void setRejectUnaccelerated(boolean rejectUnaccelerated) {
+ this.rejectUnaccelerated = rejectUnaccelerated;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (caseSensitive ? 1231 : 1237);
+ result = prime * result + ((locale == null) ? 0 :
locale.hashCode());
+ result = prime * result + maxDeterminizedStates;
+ result = prime * result + maxExpand;
+ result = prime * result + maxInspect;
+ result = prime * result + maxNgramsExtracted;
+ result = prime * result + maxStatesTraced;
+ result = prime * result + (rejectUnaccelerated ? 1231 : 1237);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Settings other = (Settings) obj;
+ if (caseSensitive != other.caseSensitive)
+ return false;
+ if (locale == null) {
+ if (other.locale != null)
+ return false;
+ } else if (!locale.equals(other.locale))
+ return false;
+ if (maxDeterminizedStates != other.maxDeterminizedStates)
+ return false;
+ if (maxExpand != other.maxExpand)
+ return false;
+ if (maxInspect != other.maxInspect)
+ return false;
+ if (maxNgramsExtracted != other.maxNgramsExtracted)
+ return false;
+ if (maxStatesTraced != other.maxStatesTraced)
+ return false;
+ if (rejectUnaccelerated != other.rejectUnaccelerated)
+ return false;
+ return true;
+ }
+ }
}
diff --git
a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterBuilder.java
b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterBuilder.java
index d0ab353..5bab331 100644
---
a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterBuilder.java
+++
b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterBuilder.java
@@ -3,6 +3,7 @@
import java.io.IOException;
import java.util.Locale;
+import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.BaseFilterBuilder;
@@ -15,17 +16,11 @@
private Boolean loadFromSource;
private String ngramField;
private Integer gramSize;
- private Integer maxExpand;
- private Integer maxStatesTraced;
- private Integer maxDeterminizedStates;
- private Integer maxNgramsExtracted;
- private Integer maxInspect;
- private Boolean caseSensitive;
- private Locale locale;
- private Boolean rejectUnaccelerated;
+ private final Settings settings = new Settings();
/**
* Start building.
+ *
* @param field the field to load and run the regex against
* @param regex the regex to run
*/
@@ -46,7 +41,7 @@
/**
* @param ngramField field containing ngrams used to prefilter checked
- * documents. If not set then no ngram acceleration is
performed.
+ * documents. If not set then no ngram acceleration is
performed.
* @return this for chaining
*/
public SourceRegexFilterBuilder ngramField(String ngramField) {
@@ -64,8 +59,17 @@
return this;
}
+ /**
+ * @param maxExpand Maximum size of range transitions to expand into
+ * single transitions when turning the automaton from the
+ * regex into an acceleration automaton. Its roughly
+ * analogous to the number of characters in a character class
+ * before it is considered a wildcard for optimization
+ * purposes.
+ * @return this for chaining
+ */
public SourceRegexFilterBuilder maxExpand(int maxExpand) {
- this.maxExpand = maxExpand;
+ settings.maxExpand(maxExpand);
return this;
}
@@ -80,7 +84,7 @@
* @return this for chaining
*/
public SourceRegexFilterBuilder maxStatesTraced(int maxStatesTraced) {
- this.maxStatesTraced = maxStatesTraced;
+ settings.maxStatesTraced(maxStatesTraced);
return this;
}
@@ -93,42 +97,43 @@
* @return this for chaining
*/
public SourceRegexFilterBuilder maxDeterminizedStates(int
maxDeterminizedStates) {
- this.maxDeterminizedStates = maxDeterminizedStates;
+ settings.maxDeterminizedStates(maxDeterminizedStates);
return this;
}
/**
* @param maxNgramsExtracted the maximum number of ngrams extracted from
the
- * regex. This is pretty much the maximum number of term
queries that
- * are exectued per regex. If any more are required to
accurately
- * limit the regex to some document set they are all assumed to
match
- * all documents that match so far. Its crude, but it limits
the number
- * of term queries while degrading reasonably well.
+ * regex. This is pretty much the maximum number of term queries
+ * that are exectued per regex. If any more are required to
+ * accurately limit the regex to some document set they are all
+ * assumed to match all documents that match so far. Its crude,
+ * but it limits the number of term queries while degrading
+ * reasonably well.
* @return this for chaining
*/
public SourceRegexFilterBuilder maxNgramsExtracted(int maxNgramsExtracted)
{
- this.maxNgramsExtracted = maxNgramsExtracted;
+ settings.maxNgramsExtracted(maxNgramsExtracted);
return this;
}
/**
* @param maxInspect the maximum number of source documents to run the
regex
* against per shard. All others after that are assumed not to
- * match. Defaults to Integer.MAX_VALUE.
+ * match. Defaults to Integer.MAX_VALUE.
* @return this for chaining
*/
public SourceRegexFilterBuilder maxInspect(int maxInspect) {
- this.maxInspect = maxInspect;
+ settings.maxInspect(maxInspect);
return this;
}
public SourceRegexFilterBuilder caseSensitive(boolean caseSensitive) {
- this.caseSensitive = caseSensitive;
+ settings.caseSensitive(caseSensitive);
return this;
}
public SourceRegexFilterBuilder locale(Locale locale) {
- this.locale = locale;
+ settings.locale(locale);
return this;
}
@@ -138,7 +143,7 @@
* @return this for chaining
*/
public SourceRegexFilterBuilder rejectUnaccelerated(boolean
rejectUnaccelerated) {
- this.rejectUnaccelerated = rejectUnaccelerated;
+ settings.rejectUnaccelerated(rejectUnaccelerated);
return this;
}
@@ -157,31 +162,138 @@
if (gramSize != null) {
builder.field("gram_size", gramSize);
}
- if (maxExpand != null) {
- builder.field("max_expand", maxExpand);
- }
- if (maxStatesTraced != null) {
- builder.field("max_states_traced", maxStatesTraced);
- }
- if (maxDeterminizedStates != null) {
- builder.field("max_determinized_states", maxDeterminizedStates);
- }
- if (maxNgramsExtracted != null) {
- builder.field("max_ngrams_extracted", maxNgramsExtracted);
- }
- if (maxInspect != null) {
- builder.field("max_inspect", maxInspect);
- }
- if (caseSensitive != null) {
- builder.field("case_sensitive", caseSensitive);
- }
- if (locale != null) {
- builder.field("locale", locale);
- }
- if (rejectUnaccelerated != null) {
- builder.field("reject_unaccelerated", rejectUnaccelerated);
- }
+ settings.innerXContent(builder, params);
builder.endObject();
}
+
+ /**
+ * Field independent settings for the SourceRegexFilter.
+ */
+ public static class Settings implements ToXContent {
+ private Integer maxExpand;
+ private Integer maxStatesTraced;
+ private Integer maxDeterminizedStates;
+ private Integer maxNgramsExtracted;
+ private Integer maxInspect;
+ private Boolean caseSensitive;
+ private Locale locale;
+ private Boolean rejectUnaccelerated;
+
+ /**
+ * @param maxExpand Maximum size of range transitions to expand into
+ * single transitions when turning the automaton from the
+ * regex into an acceleration automaton. Its roughly
+ * analogous to the number of characters in a character
class
+ * before it is considered a wildcard for optimization
+ * purposes.
+ * @return this for chaining
+ */
+ public Settings maxExpand(int maxExpand) {
+ this.maxExpand = maxExpand;
+ return this;
+ }
+
+ /**
+ * @param maxDeterminizedStates the maximum number of automaton states
+ * that Lucene will create at a time when compiling the
regex
+ * to a DFA. Higher numbers allow the regex compilation
phase
+ * to run for longer and use more memory needed to compile
+ * more complex regexes.
+ * @return this for chaining
+ */
+ public Settings maxStatesTraced(int maxStatesTraced) {
+ this.maxStatesTraced = maxStatesTraced;
+ return this;
+ }
+
+ /**
+ * @param maxDeterminizedStates the maximum number of automaton states
+ * that Lucene will create at a time when compiling the
regex
+ * to a DFA. Higher numbers allow the regex compilation
phase
+ * to run for longer and use more memory needed to compile
+ * more complex regexes.
+ * @return this for chaining
+ */
+ public Settings maxDeterminizedStates(int maxDeterminizedStates) {
+ this.maxDeterminizedStates = maxDeterminizedStates;
+ return this;
+ }
+
+ /**
+ * @param maxNgramsExtracted the maximum number of ngrams extracted
from
+ * the regex. This is pretty much the maximum number of term
+ * queries that are exectued per regex. If any more are
+ * required to accurately limit the regex to some document
+ * set they are all assumed to match all documents that
match
+ * so far. Its crude, but it limits the number of term
+ * queries while degrading reasonably well.
+ * @return this for chaining
+ */
+ public Settings maxNgramsExtracted(int maxNgramsExtracted) {
+ this.maxNgramsExtracted = maxNgramsExtracted;
+ return this;
+ }
+
+ /**
+ * @param maxInspect the maximum number of source documents to run the
+ * regex against per shard. All others after that are
assumed
+ * not to match. Defaults to Integer.MAX_VALUE.
+ * @return this for chaining
+ */
+ public Settings maxInspect(int maxInspect) {
+ this.maxInspect = maxInspect;
+ return this;
+ }
+
+ public Settings caseSensitive(boolean caseSensitive) {
+ this.caseSensitive = caseSensitive;
+ return this;
+ }
+
+ public Settings locale(Locale locale) {
+ this.locale = locale;
+ return this;
+ }
+
+ public Settings rejectUnaccelerated(boolean rejectUnaccelerated) {
+ this.rejectUnaccelerated = rejectUnaccelerated;
+ return this;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params
params) throws IOException {
+ builder.startObject();
+ innerXContent(builder, params);
+ return builder.endObject();
+ }
+
+ public XContentBuilder innerXContent(XContentBuilder builder, Params
params) throws IOException {
+ if (maxExpand != null) {
+ builder.field("max_expand", maxExpand);
+ }
+ if (maxStatesTraced != null) {
+ builder.field("max_states_traced", maxStatesTraced);
+ }
+ if (maxDeterminizedStates != null) {
+ builder.field("max_determinized_states",
maxDeterminizedStates);
+ }
+ if (maxNgramsExtracted != null) {
+ builder.field("max_ngrams_extracted", maxNgramsExtracted);
+ }
+ if (maxInspect != null) {
+ builder.field("max_inspect", maxInspect);
+ }
+ if (caseSensitive != null) {
+ builder.field("case_sensitive", caseSensitive);
+ }
+ if (locale != null) {
+ builder.field("locale", locale);
+ }
+ if (rejectUnaccelerated != null) {
+ builder.field("reject_unaccelerated", rejectUnaccelerated);
+ }
+ return builder;
+ }
+ }
}
diff --git
a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterParser.java
b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterParser.java
index bb63492..346ef9d 100644
---
a/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterParser.java
+++
b/src/main/java/org/wikimedia/search/extra/regex/SourceRegexFilterParser.java
@@ -1,7 +1,6 @@
package org.wikimedia.search.extra.regex;
import java.io.IOException;
-import java.util.Locale;
import org.apache.lucene.search.Filter;
import org.elasticsearch.common.util.LocaleUtils;
@@ -10,6 +9,7 @@
import org.elasticsearch.index.query.FilterParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryParsingException;
+import org.wikimedia.search.extra.regex.SourceRegexFilter.Settings;
import org.wikimedia.search.extra.util.FieldValues;
/**
@@ -30,15 +30,8 @@
String fieldPath = null;
FieldValues.Loader loader = FieldValues.loadFromSource();
String ngramFieldPath = null;
- int gramSize = 3;
- int maxExpand = 4;
- int maxStatesTraced = 10000;
- int maxDeterminizedStates = 20000;
- int maxNgramsExtracted = 100;
- int maxInspect = Integer.MAX_VALUE;
- boolean caseSensitive = false;
- Locale locale = Locale.ROOT;
- boolean rejectUnaccelerated = false;
+ int ngramGramSize = 3;
+ Settings settings = new Settings();
// Stuff all filters have
String filterName = null;
@@ -73,51 +66,22 @@
break;
case "gram_size":
case "gramSize":
- gramSize = parser.intValue();
- break;
- case "max_expand":
- case "maxExpand":
- maxExpand = parser.intValue();
- break;
- case "max_states_traced":
- case "maxStatesTraced":
- maxStatesTraced = parser.intValue();
- break;
- case "max_inspect":
- case "maxInspect":
- maxInspect = parser.intValue();
- break;
- case "max_determinized_states":
- case "maxDeterminizedStates":
- maxDeterminizedStates = parser.intValue();
- break;
- case "max_ngrams_extracted":
- case "maxNgramsExtracted":
- case "maxNGramsExtracted":
- maxNgramsExtracted = parser.intValue();
- break;
- case "case_sensitive":
- case "caseSensitive":
- caseSensitive = parser.booleanValue();
- break;
- case "locale":
- locale = LocaleUtils.parse(parser.text());
- break;
- case "reject_unaccelerated":
- case "rejectUnaccelerated":
- rejectUnaccelerated = parser.booleanValue();
+ ngramGramSize = parser.intValue();
break;
case "_cache":
cache = parser.booleanValue();
break;
case "_name":
filterName = parser.text();
- break;
+ break;
case "_cache_key":
case "_cacheKey":
cacheKey = new CacheKeyFilter.Key(parser.text());
break;
default:
+ if (parseInto(settings, currentFieldName, parser)) {
+ continue;
+ }
throw new QueryParsingException(parseContext.index(),
"[source-regex] filter does not support [" + currentFieldName
+ "]");
}
@@ -130,8 +94,7 @@
if (fieldPath == null) {
throw new QueryParsingException(parseContext.index(),
"[source-regex] filter must specify [field]");
}
- Filter filter = new SourceRegexFilter(fieldPath, loader, regex,
ngramFieldPath, gramSize, maxExpand, maxStatesTraced,
- maxDeterminizedStates, maxNgramsExtracted, maxInspect,
caseSensitive, locale, rejectUnaccelerated);
+ Filter filter = new SourceRegexFilter(fieldPath, ngramFieldPath,
regex, loader, settings, ngramGramSize);
if (cache) {
filter = parseContext.cacheFilter(filter, cacheKey);
}
@@ -140,4 +103,49 @@
}
return filter;
}
+
+ /**
+ * Parse a field into a settings object.
+ *
+ * @return true if the field belonged to settings, false if it didn't
+ */
+ public static boolean parseInto(SourceRegexFilter.Settings settings,
String fieldName, XContentParser parser) throws IOException {
+ switch (fieldName) {
+ case "max_expand":
+ case "maxExpand":
+ settings.setMaxExpand(parser.intValue());
+ break;
+ case "max_states_traced":
+ case "maxStatesTraced":
+ settings.setMaxStatesTraced(parser.intValue());
+ break;
+ case "max_inspect":
+ case "maxInspect":
+ settings.setMaxInspect(parser.intValue());
+ break;
+ case "max_determinized_states":
+ case "maxDeterminizedStates":
+ settings.setMaxDeterminizedStates(parser.intValue());
+ break;
+ case "max_ngrams_extracted":
+ case "maxNgramsExtracted":
+ case "maxNGramsExtracted":
+ settings.setMaxNgramsExtracted(parser.intValue());
+ break;
+ case "case_sensitive":
+ case "caseSensitive":
+ settings.setCaseSensitive(parser.booleanValue());
+ break;
+ case "locale":
+ settings.setLocale(LocaleUtils.parse(parser.text()));
+ break;
+ case "reject_unaccelerated":
+ case "rejectUnaccelerated":
+ settings.setRejectUnaccelerated(parser.booleanValue());
+ break;
+ default:
+ return false;
+ }
+ return true;
+ }
}
diff --git
a/src/main/java/org/wikimedia/search/extra/regex/UnableToAccelerateRegexException.java
b/src/main/java/org/wikimedia/search/extra/regex/UnableToAccelerateRegexException.java
index 4c96d3e..e97f22d 100644
---
a/src/main/java/org/wikimedia/search/extra/regex/UnableToAccelerateRegexException.java
+++
b/src/main/java/org/wikimedia/search/extra/regex/UnableToAccelerateRegexException.java
@@ -10,6 +10,6 @@
private static final long serialVersionUID = 2685216158813374775L;
public UnableToAccelerateRegexException(String regex, int gramSize, String
ngramField) {
- super(String.format(Locale.ROOT, "Unable to accelerate %s with %s
sized grams stored in %s", regex, gramSize, ngramField));
+ super(String.format(Locale.ROOT, "Unable to accelerate \"%s\" with %s
sized grams stored in %s", regex, gramSize, ngramField));
}
}
diff --git
a/src/main/java/org/wikimedia/search/extra/regex/ngram/NGramExtractor.java
b/src/main/java/org/wikimedia/search/extra/regex/ngram/NGramExtractor.java
index d09cecb..c1a4d98 100644
--- a/src/main/java/org/wikimedia/search/extra/regex/ngram/NGramExtractor.java
+++ b/src/main/java/org/wikimedia/search/extra/regex/ngram/NGramExtractor.java
@@ -18,7 +18,7 @@
*
* @param gramSize size of the ngram. The "n" in ngram.
* @param maxExpand Maximum size of range transitions to expand into single
- * transitions. Its roughly analogous to the number of character
+ * transitions. Its roughly analogous to the number of
characters
* in a character class before it is considered a wildcard for
* optimization purposes.
* @param maxStatesTraced maximum number of states traced during automaton
diff --git a/src/main/java/org/wikimedia/search/extra/util/FieldValues.java
b/src/main/java/org/wikimedia/search/extra/util/FieldValues.java
index dc16f68..171d7cc 100644
--- a/src/main/java/org/wikimedia/search/extra/util/FieldValues.java
+++ b/src/main/java/org/wikimedia/search/extra/util/FieldValues.java
@@ -34,7 +34,7 @@
* into Lucene every time.
*/
public static FieldValues.Loader loadFromSource() {
- return new Source();
+ return Source.INSTANCE;
}
/**
@@ -43,7 +43,7 @@
* call down into Lucene every time.
*/
public static FieldValues.Loader loadFromStoredField() {
- return new Stored();
+ return Stored.INSTANCE;
}
/**
@@ -58,6 +58,7 @@
}
private static class Source implements FieldValues.Loader {
+ private static final FieldValues.Loader INSTANCE = new Source();
@Override
public List<String> load(String path, IndexReader reader, int docId)
throws IOException {
JustSourceFieldsVisitor visitor = new JustSourceFieldsVisitor();
@@ -69,6 +70,7 @@
}
private static class Stored implements FieldValues.Loader {
+ private static final FieldValues.Loader INSTANCE = new Stored();
@Override
public List<String> load(String path, IndexReader reader, int docId)
throws IOException {
CustomFieldsVisitor visitor = new
CustomFieldsVisitor(ImmutableSet.of(path), false);
--
To view, visit https://gerrit.wikimedia.org/r/215913
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9e93a1a6c0e7e4dab9e9ca46d23ec24b325a0887
Gerrit-PatchSet: 1
Gerrit-Project: search/extra
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
