Repository: carbondata Updated Branches: refs/heads/master 34cb55194 -> b6f29dec4
[CARBONDATA-1826] Spark 2.2: Describe table & Describe Formatted shows the same result Carbon 1.3.0 - Spark 2.2: Describe table & Describe Formatted shows the same result. This closes #1630 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6f29dec Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6f29dec Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6f29dec Branch: refs/heads/master Commit: b6f29dec4335d536fe371ba985fe2c9b98cb2772 Parents: 34cb551 Author: dhatchayani <[email protected]> Authored: Thu Dec 7 12:50:05 2017 +0530 Committer: manishgupta88 <[email protected]> Committed: Wed Dec 13 09:55:49 2017 +0530 ---------------------------------------------------------------------- .../describeTable/TestDescribeTable.scala | 8 +++-- .../spark/util/CarbonReflectionUtils.scala | 8 +++++ .../sql/execution/strategy/DDLStrategy.scala | 33 +++++++++++++------- 3 files changed, 35 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala index b2a1120..a87292f 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala @@ -16,6 +16,7 @@ */ package org.apache.carbondata.spark.testsuite.describeTable +import org.apache.spark.sql.Row import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll @@ -34,8 +35,11 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll { sql("CREATE TABLE Desc2(Dec2Col1 BigInt, Dec2Col2 String, Dec2Col3 Bigint, Dec2Col4 Decimal) stored by 'carbondata'") } - ignore("test describe table") { - checkAnswer(sql("DESC Desc1"), sql("DESC Desc2")) + test("test describe table") { + checkAnswer(sql("DESC Desc1"), Seq(Row("dec2col1","bigint",null), + Row("dec2col2","string",null), + Row("dec2col3","bigint",null), + Row("dec2col4","decimal(10,0)",null))) } test("test describe formatted table") { http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala index d88f190..6cc7ae1 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala @@ -175,6 +175,14 @@ object CarbonReflectionUtils { } } + def getDescribeTableFormattedField[T: TypeTag : reflect.ClassTag](obj: T): Boolean = { + val im = rm.reflect(obj) + val isFormatted = im.symbol.typeSignature.members + .find(_.name.toString.equalsIgnoreCase("isFormatted")) + .map(l => im.reflectField(l.asTerm).get).getOrElse("false").asInstanceOf[Boolean] + isFormatted + } + def createObject(className: String, conArgs: Object*): (Any, Class[_]) = { val clazz = Utils.classForName(className) val ctor = clazz.getConstructors.head http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala index 9ae1979..8745900 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedComm import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand} import org.apache.spark.sql.CarbonExpressions.{CarbonDescribeTable => DescribeTableCommand} import org.apache.spark.sql.execution.datasources.RefreshTable -import org.apache.spark.util.FileUtils +import org.apache.spark.util.{CarbonReflectionUtils, FileUtils} import org.apache.carbondata.common.logging.{LogService, LogServiceFactory} import org.apache.carbondata.core.util.CarbonProperties @@ -134,17 +134,26 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy { } else { throw new MalformedCarbonCommandException("Unsupported alter operation on hive table") } - case desc@DescribeTableCommand(identifier, partitionSpec, isExtended) - if CarbonEnv.getInstance(sparkSession).carbonMetastore - .tableExists(identifier)(sparkSession) => - val resolvedTable = - sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed - val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan - ExecutedCommandExec( - CarbonDescribeFormattedCommand( - resultPlan, - plan.output, - identifier)) :: Nil + case desc@DescribeTableCommand(identifier, partitionSpec, isExtended) => + val isFormatted: Boolean = if (sparkSession.version.startsWith("2.1")) { + CarbonReflectionUtils + .getDescribeTableFormattedField(desc.asInstanceOf[DescribeTableCommand]) + } else { + false + } + if (CarbonEnv.getInstance(sparkSession).carbonMetastore + .tableExists(identifier)(sparkSession) && (isExtended || isFormatted)) { + val resolvedTable = + sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed + val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan + ExecutedCommandExec( + CarbonDescribeFormattedCommand( + resultPlan, + plan.output, + identifier)) :: Nil + } else { + Nil + } case ShowPartitionsCommand(t, cols) => val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore .tableExists(t)(sparkSession)
