Repository: carbondata Updated Branches: refs/heads/master e3eb03054 -> 786db2171
[CARBONDATA-2960] SDK Reader fix with projection columns SDK Reader was not working when all projection columns were given. Added exception for Complex child projections too. Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/786db217 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/786db217 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/786db217 Branch: refs/heads/master Commit: 786db217120e1d341e9a7d00ce9576dccd1d96af Parents: e3eb030 Author: Manish Nalla <[email protected]> Authored: Fri Sep 21 19:24:01 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Tue Sep 25 12:38:52 2018 +0530 ---------------------------------------------------------------------- .../hadoop/api/CarbonInputFormat.java | 13 ++++- ...tNonTransactionalCarbonTableForMapType.scala | 53 ++++++++++++++++++++ .../sdk/file/CarbonReaderBuilder.java | 8 +++ 3 files changed, 73 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/786db217/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java index 8183335..db93cbd 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java @@ -775,9 +775,20 @@ m filterExpression public String[] projectAllColumns(CarbonTable carbonTable) { List<ColumnSchema> colList = carbonTable.getTableInfo().getFactTable().getListOfColumns(); List<String> projectColumn = new ArrayList<>(); + // childCount will recursively count the number of children for any parent + // complex type and add just the parent column name while skipping the child columns. + int childDimCount = 0; for (ColumnSchema cols : colList) { if (cols.getSchemaOrdinal() != -1) { - projectColumn.add(cols.getColumnName()); + if (childDimCount == 0) { + projectColumn.add(cols.getColumnName()); + } + if (childDimCount > 0) { + childDimCount--; + } + if (cols.getDataType().isComplexType()) { + childDimCount += cols.getNumberOfChild(); + } } } String[] projectionColumns = new String[projectColumn.size()]; http://git-wip-us.apache.org/repos/asf/carbondata/blob/786db217/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableForMapType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableForMapType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableForMapType.scala index a6bc224..b060ec1 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableForMapType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableForMapType.scala @@ -20,15 +20,20 @@ package org.apache.carbondata.spark.testsuite.createTable import java.io.File import org.apache.commons.io.FileUtils +import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.Row import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll +import org.apache.carbondata.sdk.file.CarbonReader + /** * test cases for SDK complex map data type support */ class TestNonTransactionalCarbonTableForMapType extends QueryTest with BeforeAndAfterAll { + private val conf: Configuration = new Configuration(false) + private val nonTransactionalCarbonTable = new TestNonTransactionalCarbonTable private val writerPath = nonTransactionalCarbonTable.writerPath @@ -401,6 +406,54 @@ class TestNonTransactionalCarbonTableForMapType extends QueryTest with BeforeAnd dropSchema } + test("SDK Reader Without Projection Columns"){ + deleteDirectory(writerPath) + val mySchema = + """ + |{ + | "name": "address", + | "type": "record", + | "fields": [ + | { + | "name": "name", + | "type": "string" + | }, + | { + | "name": "age", + | "type": "int" + | }, + | { + | "name": "arrayRecord", + | "type": { + | "type": "array", + | "items": { + | "name": "houseDetails", + | "type": "map", + | "values": "string" + | } + | } + | } + | ] + |} + """.stripMargin + val json = """ {"name":"bob", "age":10, "arrayRecord": [{"101": "Rahul", "102": "Pawan"}]} """.stripMargin + nonTransactionalCarbonTable.WriteFilesWithAvroWriter(2, mySchema, json) + + val reader = CarbonReader.builder(writerPath, "_temp").isTransactionalTable(false).build(conf) + reader.close() + val exception1 = intercept[Exception] { + val reader1 = CarbonReader.builder(writerPath, "_temp") + .projection(Array[String] { "arrayRecord.houseDetails" }).isTransactionalTable(false) + .build(conf) + reader1.close() + } + assert(exception1.getMessage + .contains( + "Complex child columns projection NOT supported through CarbonReader")) + println("Done test") + } + + test("Read sdk writer Avro output Map Type") { buildMapSchema(3) assert(new File(writerPath).exists()) http://git-wip-us.apache.org/repos/asf/carbondata/blob/786db217/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java index 15066ce..3ae3d59 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java @@ -205,6 +205,14 @@ public class CarbonReaderBuilder { if (projectionColumns != null) { // set the user projection + int len = projectionColumns.length; + // TODO : Handle projection of complex child columns + for (int i = 0; i < len; i++) { + if (projectionColumns[i].contains(".")) { + throw new UnsupportedOperationException( + "Complex child columns projection NOT supported through CarbonReader"); + } + } format.setColumnProjection(job.getConfiguration(), projectionColumns); }
