Repository: carbondata Updated Branches: refs/heads/master f7552a97d -> 64ae5ae0b
[CARBONDATA-2634][BloomDataMap] Add datamap properties in show datamap outputs add datamap properties in show datamap outputs This closes #2404 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/64ae5ae0 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/64ae5ae0 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/64ae5ae0 Branch: refs/heads/master Commit: 64ae5ae0bbd968750c1db97fc2ff5b2bf0d60976 Parents: f7552a9 Author: xuchuanyin <[email protected]> Authored: Sun Jun 24 11:49:27 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Fri Jun 29 11:10:56 2018 +0800 ---------------------------------------------------------------------- .../schema/datamap/DataMapProperty.java | 8 +++ .../core/metadata/schema/table/TableSchema.java | 6 +- .../testsuite/datamap/TestDataMapCommand.scala | 63 ++++++++++++++++++++ .../datamap/CarbonDataMapShowCommand.scala | 28 ++++++--- .../preaaggregate/PreAggregateUtil.scala | 3 +- 5 files changed, 97 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java index 9bd78da..37c051a 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java @@ -36,4 +36,12 @@ public class DataMapProperty { * property internally */ public static final String DEFERRED_REBUILD = "_internal.deferred.rebuild"; + /** + * for internal property 'CHILD_SELECT_QUERY' + */ + public static final String CHILD_SELECT_QUERY = "CHILD_SELECT QUERY"; + /** + * for internal property 'QUERYTYPE' + */ + public static final String QUERY_TYPE = "QUERYTYPE"; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java index 3d9e068..b53a9d8 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java @@ -30,6 +30,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.schema.BucketingInfo; import org.apache.carbondata.core.metadata.schema.PartitionInfo; import org.apache.carbondata.core.metadata.schema.SchemaEvolution; +import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.core.util.CarbonUtil; @@ -279,13 +280,12 @@ public class TableSchema implements Serializable, Writable { new RelationIdentifier(databaseName, tableName, tableId); Map<String, String> properties = new HashMap<>(); if (queryString != null) { - properties.put( - "CHILD_SELECT QUERY", + properties.put(DataMapProperty.CHILD_SELECT_QUERY, CarbonUtil.encodeToString(queryString.trim().getBytes( // replace = to with & as hive metastore does not allow = inside. For base 64 // only = is allowed as special character , so replace with & CarbonCommonConstants.DEFAULT_CHARSET)).replace("=", "&")); - properties.put("QUERYTYPE", queryType); + properties.put(DataMapProperty.QUERY_TYPE, queryType); } DataMapSchema dataMapSchema = new DataMapSchema(dataMapName, className); dataMapSchema.setProperties(properties); http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala index 642607c..afca3b2 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala @@ -204,6 +204,69 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { } } + test("test show datamap: show datamap property related information") { + val tableName = "datamapshowtest" + val datamapName = "bloomdatamap" + val datamapName2 = "bloomdatamap2" + val datamapName3 = "bloomdatamap3" + sql(s"drop table if exists $tableName") + // for index datamap + sql(s"create table $tableName (a string, b string, c string) stored by 'carbondata'") + sql( + s""" + | create datamap $datamapName on table $tableName using 'bloomfilter' + | DMPROPERTIES ('index_columns'='a', 'bloom_size'='32000', 'bloom_fpp'='0.001') + """.stripMargin) + sql( + s""" + | create datamap $datamapName2 on table $tableName using 'bloomfilter' + | DMPROPERTIES ('index_columns'='b') + """.stripMargin) + sql( + s""" + | create datamap $datamapName3 on table $tableName using 'bloomfilter' + | with deferred rebuild + | DMPROPERTIES ('index_columns'='c') + """.stripMargin) + var result = sql(s"show datamap on table $tableName").cache() + checkAnswer(sql(s"show datamap on table $tableName"), + Seq(Row(datamapName, "bloomfilter", s"default.$tableName", "'bloom_fpp'='0.001', 'bloom_size'='32000', 'index_columns'='a'"), + Row(datamapName2, "bloomfilter", s"default.$tableName", "'index_columns'='b'"), + Row(datamapName3, "bloomfilter", s"default.$tableName", "'index_columns'='c'"))) + result.unpersist() + sql(s"drop table if exists $tableName") + + // for timeseries datamap + sql(s"CREATE TABLE $tableName(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'") + sql( + s""" + | CREATE DATAMAP agg0_hour ON TABLE $tableName + | USING 'timeSeries' + | DMPROPERTIES ( + | 'EVENT_TIME'='mytime', + | 'HOUR_GRANULARITY'='1') + | AS SELECT mytime, SUM(age) FROM $tableName + | GROUP BY mytime + """.stripMargin) + checkAnswer(sql(s"show datamap on table $tableName"), + Seq(Row("agg0_hour", "timeSeries", s"default.${tableName}_agg0_hour", "'event_time'='mytime', 'hour_granularity'='1'"))) + sql(s"drop table if exists $tableName") + + // for preaggreate datamap, the property is empty + sql(s"CREATE TABLE $tableName(id int, name string, city string, age string)" + + s" STORED BY 'org.apache.carbondata.format'") + sql ( + s""" + | CREATE DATAMAP agg0 ON TABLE $tableName USING 'preaggregate' AS + | SELECT name, + | count(age) + | FROM $tableName GROUP BY name + | """.stripMargin) + checkAnswer(sql(s"show datamap on table $tableName"), + Seq(Row("agg0", "preaggregate", s"default.${tableName}_agg0", ""))) + sql(s"drop table if exists $tableName") + } + test("test if preaggregate load is successfull for hivemetastore") { try { CarbonProperties.getInstance() http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala index 844d4c0..3ee8e67 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.spark.sql.types.StringType import org.apache.carbondata.core.datamap.DataMapStoreManager -import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider +import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty} import org.apache.carbondata.core.metadata.schema.table.DataMapSchema /** @@ -42,7 +42,8 @@ case class CarbonDataMapShowCommand(tableIdentifier: Option[TableIdentifier]) override def output: Seq[Attribute] = { Seq(AttributeReference("DataMapName", StringType, nullable = false)(), AttributeReference("ClassName", StringType, nullable = false)(), - AttributeReference("Associated Table", StringType, nullable = false)()) + AttributeReference("Associated Table", StringType, nullable = false)(), + AttributeReference("DataMap Properties", StringType, nullable = false)()) } override def processData(sparkSession: SparkSession): Seq[Row] = { @@ -66,11 +67,24 @@ case class CarbonDataMapShowCommand(tableIdentifier: Option[TableIdentifier]) private def convertToRow(schemaList: util.List[DataMapSchema]) = { if (schemaList != null && schemaList.size() > 0) { - schemaList.asScala.map { s => - var table = "(NA)" - val relationIdentifier = s.getRelationIdentifier - table = relationIdentifier.getDatabaseName + "." + relationIdentifier.getTableName - Row(s.getDataMapName, s.getProviderName, table) + schemaList.asScala + .map { s => + val relationIdentifier = s.getRelationIdentifier + val table = relationIdentifier.getDatabaseName + "." + relationIdentifier.getTableName + // preaggregate datamap does not support user specified property, therefor we return empty + val dmPropertieStr = if (s.getProviderName.equalsIgnoreCase( + DataMapClassProvider.PREAGGREGATE.getShortName)) { + "" + } else { + s.getProperties.asScala + // ignore internal used property + .filter(p => !p._1.equalsIgnoreCase(DataMapProperty.DEFERRED_REBUILD) && + !p._1.equalsIgnoreCase(DataMapProperty.CHILD_SELECT_QUERY) && + !p._1.equalsIgnoreCase(DataMapProperty.QUERY_TYPE)) + .map(p => s"'${ p._1 }'='${ p._2 }'").toSeq + .sorted.mkString(", ") + } + Row(s.getDataMapName, s.getProviderName, table, dmPropertieStr) } } else { Seq.empty http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala index a7bcddb..06ebc43 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala @@ -40,6 +40,7 @@ import org.apache.carbondata.common.logging.{LogService, LogServiceFactory} import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage} import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl +import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema, TableSchema} import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema import org.apache.carbondata.core.util.CarbonUtil @@ -869,7 +870,7 @@ object PreAggregateUtil { def getChildQuery(aggDataMapSchema: AggregationDataMapSchema): String = { new String( CarbonUtil.decodeStringToBytes( - aggDataMapSchema.getProperties.get("CHILD_SELECT QUERY").replace("&", "=")), + aggDataMapSchema.getProperties.get(DataMapProperty.CHILD_SELECT_QUERY).replace("&", "=")), CarbonCommonConstants.DEFAULT_CHARSET) }
