Problem: Preaggregate select queries require table path of parent table to access dictionary files. Therefore in executor CarbonMetadata class was used to get parent table object. As CarbonMetadata class is only meant to be used in driver and is not filled with carbontable objects for select queries therefore the query was throwing NPE.
Solution: Pass parent table path from driver to executor by adding a new variable in RelationIdentifier. This will not be written to thrift, instead will be used to carry tablePath property from driver to executor. This closes #2786 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ca985f0 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ca985f0 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ca985f0 Branch: refs/heads/branch-1.5 Commit: 9ca985f0d96380d96dae6fdab2d4ee014d5ac345 Parents: 682160f Author: kunal642 <[email protected]> Authored: Fri Sep 28 17:41:32 2018 +0530 Committer: ravipesala <[email protected]> Committed: Wed Oct 3 20:02:49 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/schema/table/RelationIdentifier.java | 12 ++++++++++++ .../carbondata/core/scan/executor/util/QueryUtil.java | 10 ++-------- .../spark/sql/CarbonDatasourceHadoopRelation.scala | 12 ++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java index 9a1dad1..0e8042d 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java @@ -32,6 +32,8 @@ public class RelationIdentifier implements Serializable, Writable { private String tableId; + private String tablePath = ""; + public RelationIdentifier(String databaseName, String tableName, String tableId) { this.databaseName = databaseName; this.tableName = tableName; @@ -50,16 +52,26 @@ public class RelationIdentifier implements Serializable, Writable { return tableId; } + public String getTablePath() { + return tablePath; + } + + public void setTablePath(String tablePath) { + this.tablePath = tablePath; + } + @Override public void write(DataOutput out) throws IOException { out.writeUTF(databaseName); out.writeUTF(tableName); out.writeUTF(tableId); + out.writeUTF(tablePath); } @Override public void readFields(DataInput in) throws IOException { this.databaseName = in.readUTF(); this.tableName = in.readUTF(); this.tableId = in.readUTF(); + this.tablePath = in.readUTF(); } @Override public boolean equals(Object o) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index 9fb0857..7849d10 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -393,12 +393,6 @@ public class QueryUtil { public static AbsoluteTableIdentifier getTableIdentifierForColumn( CarbonDimension carbonDimension) { - RelationIdentifier parentRelationIdentifier = - carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0) - .getRelationIdentifier(); - String parentTablePath = CarbonMetadata.getInstance() - .getCarbonTable(parentRelationIdentifier.getDatabaseName(), - parentRelationIdentifier.getTableName()).getTablePath(); RelationIdentifier relation = carbonDimension.getColumnSchema() .getParentColumnTableRelations() .get(0) @@ -406,8 +400,8 @@ public class QueryUtil { String parentTableName = relation.getTableName(); String parentDatabaseName = relation.getDatabaseName(); String parentTableId = relation.getTableId(); - return AbsoluteTableIdentifier.from(parentTablePath, parentDatabaseName, parentTableName, - parentTableId); + return AbsoluteTableIdentifier.from(relation.getTablePath(), parentDatabaseName, + parentTableName, parentTableId); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala index 8a0404c..04ec75d 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql +import scala.collection.JavaConverters._ import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.util.control.Breaks._ @@ -35,6 +36,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.indexstore.PartitionSpec import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier import org.apache.carbondata.core.metadata.schema.table.CarbonTable +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension import org.apache.carbondata.core.scan.expression.Expression import org.apache.carbondata.core.scan.expression.logical.AndExpression import org.apache.carbondata.hadoop.CarbonProjection @@ -80,6 +82,16 @@ case class CarbonDatasourceHadoopRelation( val projection = new CarbonProjection + if (carbonTable.isChildDataMap) { + val parentTableIdentifier = carbonTable.getTableInfo.getParentRelationIdentifiers.get(0) + val path = CarbonEnv.getCarbonTable(Some(parentTableIdentifier.getDatabaseName), + parentTableIdentifier.getTableName)(sparkSession).getTablePath + for (carbonDimension: CarbonDimension <- carbonTable.getAllDimensions.asScala) { + carbonDimension.getColumnSchema.getParentColumnTableRelations.get(0) + .getRelationIdentifier.setTablePath(path) + } + } + // As Filter pushdown for Complex datatype is not supported, if filter is applied on complex // column, then Projection pushdown on Complex Columns will not take effect. Hence, check if // filter contains Struct Complex Column.
