Repository: carbondata Updated Branches: refs/heads/master 9db662a2d -> 5cad92f4f
[HOTFIX] Fix lucene match limit code Problem Currently, Lucene match limit is set as Static in MatchExpression it cannot work in concurrent scenarios. Solution: Change to object variable and get the match max limit from expression. This closes #2250 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5cad92f4 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5cad92f4 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5cad92f4 Branch: refs/heads/master Commit: 5cad92f4f23f47f6c0b660b69f2c6edad8597974 Parents: 9db662a Author: ravipesala <[email protected]> Authored: Sun Apr 29 20:49:18 2018 +0530 Committer: Jacky Li <[email protected]> Committed: Sun May 6 19:57:42 2018 +0800 ---------------------------------------------------------------------- .../core/scan/expression/MatchExpression.java | 15 ++++++++++----- .../datamap/lucene/LuceneFineGrainDataMap.java | 17 ++++++++++++++--- .../apache/spark/sql/optimizer/CarbonFilters.scala | 6 +++--- 3 files changed, 27 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java index db806a7..52104be 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java @@ -29,15 +29,20 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf; @InterfaceAudience.Internal public class MatchExpression extends Expression { private String queryString; - public static String maxDoc; + private int maxDoc; - public MatchExpression(String queryString, String maxDoc) { + public MatchExpression(String queryString) { this.queryString = queryString; - setMaxDoc(maxDoc); + this.maxDoc = -1; } - private void setMaxDoc(String maxDoc) { - MatchExpression.maxDoc = maxDoc; + public MatchExpression(String queryString, int maxDoc) { + this.queryString = queryString; + this.maxDoc = maxDoc; + } + + public int getMaxDoc() { + return maxDoc; } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java ---------------------------------------------------------------------- diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java index 86fba32..8e457b7 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java @@ -153,8 +153,19 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap { * Return Maximum records * @return */ - private int getMaxDoc() { - return Integer.parseInt(MatchExpression.maxDoc); + private int getMaxDoc(Expression expression) { + if (expression.getFilterExpressionType() == ExpressionType.TEXT_MATCH) { + int maxDoc = ((MatchExpression) expression).getMaxDoc(); + if (maxDoc < 0) { + maxDoc = Integer.MAX_VALUE; + } + return maxDoc; + } + + for (Expression child : expression.getChildren()) { + return getMaxDoc(child); + } + return Integer.MAX_VALUE; } /** @@ -172,7 +183,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap { String strQuery = getQueryString(filterExp.getFilterExpression()); int maxDocs; try { - maxDocs = getMaxDoc(); + maxDocs = getMaxDoc(filterExp.getFilterExpression()); } catch (NumberFormatException e) { maxDocs = Integer.MAX_VALUE; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5cad92f4/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala index 1e46998..07a444f 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.optimizer import java.util import scala.collection.JavaConverters._ +import scala.util.Try import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions._ @@ -30,7 +31,6 @@ import org.apache.spark.sql.CarbonEndsWith import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.hive.CarbonSessionCatalog -import org.apache.spark.sql.sources.Filter import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datamap.Segment @@ -141,9 +141,9 @@ object CarbonFilters { case FalseExpr() => Some(new FalseExpression(null)) case TextMatch(queryString) => - Some(new MatchExpression(queryString, null)) + Some(new MatchExpression(queryString)) case TextMatchLimit(queryString, maxDoc) => - Some(new MatchExpression(queryString, maxDoc)) + Some(new MatchExpression(queryString, Try(maxDoc.toInt).getOrElse(Integer.MAX_VALUE))) case _ => None } }
