Repository: carbondata
Updated Branches:
  refs/heads/master aa910ddb2 -> ac30e3e72


[CARBONDATA-2103]Make show datamaps configurable in show tables command

Make the show datamaps in show tables configurable:

a new carbon property is added called carbon.query.show.datamaps, by default is 
it true, show show tables will list all the table including main table and 
datamaps.
if we want to filter datamaps in show tables, configure this as false

This closes #2015


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ac30e3e7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ac30e3e7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ac30e3e7

Branch: refs/heads/master
Commit: ac30e3e721ab7a772a1010847aa90b33c0ebd2f6
Parents: aa910dd
Author: akashrn5 <akashnilu...@gmail.com>
Authored: Wed Feb 28 17:38:50 2018 +0530
Committer: ravipesala <ravi.pes...@gmail.com>
Committed: Fri Mar 2 20:12:19 2018 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  5 +++++
 .../preaggregate/TestPreAggCreateCommand.scala  | 21 ++++++++++++++++----
 .../command/table/CarbonShowTablesCommand.scala | 20 ++++++++++++++-----
 .../spark/sql/hive/CarbonSessionState.scala     | 17 ++++++++++++----
 .../spark/sql/hive/CarbonSessionState.scala     | 17 ++++++++++++----
 5 files changed, 63 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ac30e3e7/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 0a06abc..fa2b7d8 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1610,6 +1610,11 @@ public final class CarbonCommonConstants {
       "carbon.heap.memory.pooling.threshold.bytes";
   public static final String 
CARBON_HEAP_MEMORY_POOLING_THRESHOLD_BYTES_DEFAULT = "1048576";
 
+  @CarbonProperty
+  public static final String CARBON_SHOW_DATAMAPS = 
"carbon.query.show.datamaps";
+
+  public static final String CARBON_SHOW_DATAMAPS_DEFAULT = "true";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ac30e3e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 8b71a31..5b7c310 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -19,16 +19,18 @@ package 
org.apache.carbondata.integration.spark.testsuite.preaggregate
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.sql.{AnalysisException, CarbonDatasourceHadoopRelation}
+import org.apache.spark.sql.{AnalysisException, 
CarbonDatasourceHadoopRelation, Row}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.hive.CarbonRelation
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.encoder.Encoding
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable
 import 
org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, 
MalformedDataMapCommandException}
 
 class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
@@ -237,6 +239,7 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
 
   val timeSeries = TIMESERIES.toString
   test("remove agg tables from show table command") {
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
     sql("DROP TABLE IF EXISTS tbl_1")
     sql("DROP TABLE IF EXISTS sparktable")
     sql("create table if not exists  tbl_1(imei string,age int,mac string 
,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 
'carbondata' ")
@@ -249,6 +252,7 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
       ".eventTime'='prodate', 
'timeseries.hierarchy'='hour=1,day=1,month=1,year=1') as select prodate," +
       "mac from tbl_1 group by prodate,mac")
     checkExistence(sql("show tables"), false, 
"tbl_1_preagg_sum","tbl_1_agg2_day","tbl_1_agg2_hour","tbl_1_agg2_month","tbl_1_agg2_year")
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
   }
 
   test("test pre agg  create table 21: create with preaggregate and 
hierarchy") {
@@ -305,12 +309,14 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
   }
 
   test("remove  agg tables from show table command") {
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
     sql("DROP TABLE IF EXISTS tbl_1")
     sql("create table if not exists  tbl_1(imei string,age int,mac string 
,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 
'carbondata' ")
     sql("create datamap agg1 on table tbl_1 using 'preaggregate' as select 
mac, sum(age) from tbl_1 group by mac")
     sql("create table if not exists  sparktable(imei string,age int,mac string 
,prodate timestamp,update timestamp,gamepoint double,contrid double) ")
     checkExistence(sql("show tables"), false, "tbl_1_agg1")
     checkExistence(sql("show tables"), true, "sparktable","tbl_1")
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,
 CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
   }
 
 
@@ -380,10 +386,14 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
   }
 
   test("test show tables filterted with datamaps") {
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
     sql("create table showTables(name string, age int) stored by 'carbondata'")
-    sql("create datamap preAgg on table showTables using 'preaggregate' as 
select sum(age) from showTables")
+    sql(
+      "create datamap preAgg on table showTables using 'preaggregate' as 
select sum(age) from showTables")
     sql("show tables").show()
-    assert(!sql("show tables").collect().contains("showTables_preagg"))
+    checkExistence(sql("show tables"), false, "showtables_preagg")
+    
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
+    checkExistence(sql("show tables"), true, "showtables_preagg")
   }
 
   test("test create main and preagg table of same name in two database") {
@@ -426,7 +436,10 @@ class TestPreAggCreateCommand extends QueryTest with 
BeforeAndAfterAll {
     carbonTable
   }
 
-  override def afterAll { 
+  override def afterAll {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,
+        CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
     sql("drop database if exists otherDB cascade")
     sql("drop table if exists maintable")
     sql("drop table if exists PreAggMain")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ac30e3e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala
index e3c4e97..534703d 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala
@@ -17,9 +17,7 @@
 
 package org.apache.spark.sql.execution.command.table
 
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.expressions.{Attribute, 
AttributeReference}
 import org.apache.spark.sql.execution.command.MetadataCommand
@@ -44,13 +42,25 @@ private[sql] case class CarbonShowTablesCommand ( 
databaseName: Option[String],
     var tables =
       tableIdentifierPattern.map(catalog.listTables(db, 
_)).getOrElse(catalog.listTables(db))
     val externalCatalog = sparkSession.sharedState.externalCatalog
+    // this method checks whether the table is mainTable or datamap based on 
property "isVisible"
+    def isMainTable(tableIdent: TableIdentifier) = {
+      var isMainTable = true
+      try {
+        isMainTable = externalCatalog.getTable(db, 
tableIdent.table).storage.properties
+          .getOrElse("isVisible", true).toString.toBoolean
+      } catch {
+        case ex: Throwable =>
+        // ignore the exception for show tables
+      }
+      isMainTable
+    }
     // tables will be filtered for all the dataMaps to show only main tables
     tables.collect {
-      case tableIdent if externalCatalog.getTable(db, 
tableIdent.table).storage.properties
-        .getOrElse("isVisible", true).toString.toBoolean =>
+      case tableIdent if isMainTable(tableIdent) =>
         val isTemp = catalog.isTemporaryTable(tableIdent)
         Row(tableIdent.database.getOrElse("default"), tableIdent.table, isTemp)
     }
+
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ac30e3e7/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
 
b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
index d45020b..1b7f0cb 100644
--- 
a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ 
b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.optimizer.{CarbonIUDRule, 
CarbonLateDecodeRule, Carb
 import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, 
CarbonSpark2SqlParser, CarbonSparkSqlParser}
 import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, 
ExperimentalMethods, SparkSession, Strategy}
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datamap.DataMapStoreManager
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
 import org.apache.carbondata.core.util.CarbonProperties
@@ -332,9 +333,17 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: 
CarbonSpark2SqlParser, sparkSes
     }
   }
 
-  override def visitShowTables(ctx: ShowTablesContext): LogicalPlan = 
withOrigin(ctx) {
-    CarbonShowTablesCommand(
-      Option(ctx.db).map(_.getText),
-      Option(ctx.pattern).map(string))
+  override def visitShowTables(ctx: ShowTablesContext): LogicalPlan = {
+    withOrigin(ctx) {
+      if (CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,
+          CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT).toBoolean) {
+        super.visitShowTables(ctx)
+      } else {
+        CarbonShowTablesCommand(
+          Option(ctx.db).map(_.getText),
+          Option(ctx.pattern).map(string))
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ac30e3e7/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git 
a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
 
b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
index b9425d6..a119bda 100644
--- 
a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ 
b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -44,6 +44,7 @@ import 
org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlPa
 import org.apache.spark.sql.types.DecimalType
 import org.apache.spark.util.CarbonReflectionUtils
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datamap.DataMapStoreManager
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
 import org.apache.carbondata.core.util.CarbonProperties
@@ -391,9 +392,17 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: 
CarbonSpark2SqlParser, sparkSes
     super.visitCreateTable(ctx)
   }
 
-  override def visitShowTables(ctx: ShowTablesContext): LogicalPlan = 
withOrigin(ctx) {
-    CarbonShowTablesCommand(
-      Option(ctx.db).map(_.getText),
-      Option(ctx.pattern).map(string))
+  override def visitShowTables(ctx: ShowTablesContext): LogicalPlan = {
+    withOrigin(ctx) {
+      if (CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,
+          CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT).toBoolean) {
+        super.visitShowTables(ctx)
+      } else {
+        CarbonShowTablesCommand(
+          Option(ctx.db).map(_.getText),
+          Option(ctx.pattern).map(string))
+      }
+    }
   }
 }

Reply via email to