This is an automated email from the ASF dual-hosted git repository.
qiangcai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 65d9813 [CARBONDATA-3890] Fix MV case sensitive issues with
ImplicitCastInputTypes and add DOC for show MV
65d9813 is described below
commit 65d9813652d76bd6b0c37e360618459224a08f21
Author: Indhumathi27 <[email protected]>
AuthorDate: Fri Jul 3 21:09:49 2020 +0530
[CARBONDATA-3890] Fix MV case sensitive issues with ImplicitCastInputTypes
and add DOC for show MV
Why is this PR needed?
Issue 1:
Queries having implicitCastInputTypes expressions is not hitting mv when
expression is provided in Upper case
Issue 2:
MV document does not have info for SHOW MATERIALIZED VIEWS command
What changes were proposed in this PR?
Solution 1:
Transform castOrImplicitCastExp expression to change it's child attribute
reference name to lower case while matching subsumer and subsume
Solution 2:
Added document for SHOW MATERIALIZED VIEWS command
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3823
---
docs/mv-guide.md | 21 ++++++++++++++++++++
.../org/apache/spark/sql/optimizer/MVMatcher.scala | 17 +++++++++++-----
.../view/rewrite/TestAllOperationsOnMV.scala | 23 ++++++++++++++++++++++
3 files changed, 56 insertions(+), 5 deletions(-)
diff --git a/docs/mv-guide.md b/docs/mv-guide.md
index 24e38b1..f6478a9 100644
--- a/docs/mv-guide.md
+++ b/docs/mv-guide.md
@@ -23,6 +23,7 @@
* [Querying Data](#querying-data)
* [Compaction](#compacting)
* [Data Management](#data-management)
+* [Show Materialized Views](#show-materialized-views)
* [Time Series Support](#time-series-support)
* [Time Series RollUp Support](#time-series-rollup-support)
@@ -221,6 +222,26 @@
Basically, user can manually trigger the operation by re-building the
materialized view.
+### Show Materialized Views
+
+ Command syntax:
+ ```
+ SHOW MATERIALIZED VIEWS [ON TABLE [db_name.]table_name]
+ ```
+
+SHOW MATERIALIZED VIEWS command will display the information about all the
materialized
+views created on the database or on the carbon table.
+The current information includes:
+
+ | Column Info | Description
|
+
|-----------------------------|----------------------------------------------------------------------------|
+ | Database | Materialized view database name
|
+ | Name | Materialized view name
|
+ | Status | ENABLED / DISABLED
|
+ | Refresh Mode | FULL / INCREMENTAL refresh to MV
|
+ | Refresh Trigger Mode | ON_COMMIT / ON_MANUAL refresh to MV provided by
user |
+ | Properties | Table properties of the materialized view
|
+
## Time Series Support
Time series data are simply measurements or events that are tracked,
monitored, down sampled, and
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
index d862920..4aee148 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
@@ -93,7 +93,10 @@ private abstract class MVMatchPattern extends Logging {
case function: ScalaUDF if
function.function.isInstanceOf[TimeSeriesFunction] =>
getTransformedTimeSeriesFunction(function) -> alias.toAttribute
case cast: Cast if cast.child.isInstanceOf[AttributeReference] =>
- getTransformedCastExpression(cast) -> alias.toAttribute
+ getTransformedCastOrImplicitCastExpression(cast) ->
alias.toAttribute
+ case implicitCastInputTypeExp: ImplicitCastInputTypes =>
+
getTransformedCastOrImplicitCastExpression(implicitCastInputTypeExp) ->
+ alias.toAttribute
case _ =>
alias.child -> alias.toAttribute
}
@@ -120,7 +123,9 @@ private abstract class MVMatchPattern extends Logging {
case function: ScalaUDF if
function.function.isInstanceOf[TimeSeriesFunction] =>
getTransformedTimeSeriesFunction(function)
case cast: Cast if cast.child.isInstanceOf[AttributeReference]
=>
- getTransformedCastExpression(cast)
+ getTransformedCastOrImplicitCastExpression(cast)
+ case implicitCastInputTypeExp: ImplicitCastInputTypes =>
+
getTransformedCastOrImplicitCastExpression(implicitCastInputTypeExp)
}
attribute = aliasMapExp.get(newExp)
}
@@ -247,10 +252,12 @@ private abstract class MVMatchPattern extends Logging {
}
/**
- * transform cast expression to change it's child attribute reference name
to lower case
+ * transform castOrImplicitCastExp expression to change it's child attribute
reference name to
+ * lower case
*/
- protected def getTransformedCastExpression(cast: Cast): Expression = {
- cast.transform {
+ protected def getTransformedCastOrImplicitCastExpression(
+ castOrImplicitCastExp: Expression): Expression = {
+ castOrImplicitCastExp.transform {
case reference: AttributeReference =>
CarbonToSparkAdapter.createAttributeReference(
reference.name.toLowerCase,
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
index c1d7d0e..bb6fe56 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
@@ -678,6 +678,29 @@ class TestAllOperationsOnMV extends QueryTest with
BeforeAndAfterEach {
sql("drop table IF EXISTS maintable")
}
+ test("test case sensitive issues with implicit cast type expressions") {
+ sql("drop table IF EXISTS maintable")
+ sql("CREATE TABLE maintable (CUST_ID int,CUST_NAME
String,ACTIVE_EMUI_VERSION string, DOB " +
+ "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2
bigint,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double,
Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 int) STORED AS carbondata")
+ sql("insert into maintable select 1,'abc','2001','2017-09-01
00:00:00','2017-09-03
00:00:00',1234567,1234564,'1234.456','1234.4567',1.123455,1.123455,null")
+ sql("drop materialized view if exists mv1")
+ sql("create materialized view mv1 as select length(CUST_NAME) from
maintable where CUST_ID IS NULL or DOB IS NOT NULL or BIGINT_COLUMN1=1234567
or" +
+ " DECIMAL_COLUMN1=1234.456 or Double_COLUMN1=1.123455 or
INTEGER_COLUMN1 IS NULL")
+ val withUpperCase = sql(
+ "select length(CUST_NAME) from maintable where CUST_ID IS NULL or DOB IS
NOT NULL or " +
+ "BIGINT_COLUMN1=1234567 or DECIMAL_COLUMN1=1234.456 or
Double_COLUMN1=1.123455 or INTEGER_COLUMN1 IS NULL")
+ val withLowerCase = sql(
+ "select length(cust_name) from maintable where cust_id IS NULL or dob IS
NOT NULL or " +
+ "bigint_column1=1234567 or decimal_column1=1234.456 or
double_column1=1.123455 or integer_column1 IS NULL")
+ checkAnswer(withUpperCase, Seq(Row(3)))
+ checkAnswer(withLowerCase, Seq(Row(3)))
+ TestUtil.verifyMVHit(withUpperCase.queryExecution.optimizedPlan, "mv1")
+ TestUtil.verifyMVHit(withLowerCase.queryExecution.optimizedPlan, "mv1")
+ sql("drop table IF EXISTS maintable")
+ }
+
test("drop meta cache on mv materialized view table") {
defaultConfig()
sql("drop table IF EXISTS maintable")
