[CARBONDATA-2982] CarbonSchemaReader support array<string> This PR fix the issue and change : org.apache.carbondata.sdk.file.CarbonSchemaReader#readSchemaInDataFile org.apache.carbondata.sdk.file.CarbonSchemaReader#readSchemaInIndexFile
This PR remove child schema This closes #2780 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8a51c9b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8a51c9b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8a51c9b Branch: refs/heads/branch-1.5 Commit: d8a51c9bf314fb1cd5f6112e66eb04e776a0553d Parents: 7d1fcb3 Author: xubo245 <[email protected]> Authored: Fri Sep 28 11:47:22 2018 +0800 Committer: manishgupta88 <[email protected]> Committed: Wed Oct 3 16:24:38 2018 +0530 ---------------------------------------------------------------------- .../examples/sdk/CarbonReaderExample.java | 4 +- .../carbondata/examplesCI/RunExamples.scala | 5 ++ .../carbondata/sdk/file/CarbonSchemaReader.java | 14 +++- .../carbondata/sdk/file/CarbonReaderTest.java | 86 ++++++++++++++++++++ 4 files changed, 105 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java index 9e80567..ef4ae7a 100644 --- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java +++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java @@ -122,10 +122,11 @@ public class CarbonReaderExample { for (int j = 0; j < arr.length; j++) { System.out.print(arr[j] + " "); } + assert (arr[0].equals("Hello")); + assert (arr[3].equals("Carbon")); System.out.println(); i++; } - System.out.println("\nFinished"); reader.close(); // Read data @@ -148,7 +149,6 @@ public class CarbonReaderExample { row[5], row[6], row[7], row[8], row[9], row[10])); i++; } - System.out.println("\nFinished"); reader2.close(); FileUtils.deleteDirectory(new File(path)); } catch (Throwable e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala ---------------------------------------------------------------------- diff --git a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala index 2b9b999..6a13dc3 100644 --- a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala +++ b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala @@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.examples._ import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.sdk.CarbonReaderExample /** * Test suite for examples @@ -113,4 +114,8 @@ class RunExamples extends QueryTest with BeforeAndAfterAll { test("ExternalTableExample") { ExternalTableExample.exampleBody(spark) } + + test("CarbonReaderExample") { + CarbonReaderExample.main(null) + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java index d8882bc..e84a25a 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java @@ -65,7 +65,15 @@ public class CarbonSchemaReader { */ public static Schema readSchemaInDataFile(String dataFilePath) throws IOException { CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath); - return new Schema(reader.readSchema()); + List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); + List<ColumnSchema> schemaList = reader.readSchema(); + for (int i = 0; i < schemaList.size(); i++) { + ColumnSchema columnSchema = schemaList.get(i); + if (!(columnSchema.getColumnName().contains("."))) { + columnSchemaList.add(columnSchema); + } + } + return new Schema(columnSchemaList); } /** @@ -97,7 +105,9 @@ public class CarbonSchemaReader { List<org.apache.carbondata.format.ColumnSchema> table_columns = readIndexHeader.getTable_columns(); for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) { - columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema)); + if (!(columnSchema.column_name.contains("."))) { + columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema)); + } } return new Schema(columnSchemaList); } finally { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 8c5ffe5..8d95456 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -1435,5 +1435,91 @@ public class CarbonReaderTest extends TestCase { FileUtils.deleteDirectory(new File("./testWriteFiles")); } + @Test + public void testReadSchemaFromDataFileArrayString() { + String path = "./testWriteFiles"; + try { + FileUtils.deleteDirectory(new File(path)); + Field[] fields = new Field[11]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("shortField", DataTypes.SHORT); + fields[2] = new Field("intField", DataTypes.INT); + fields[3] = new Field("longField", DataTypes.LONG); + fields[4] = new Field("doubleField", DataTypes.DOUBLE); + fields[5] = new Field("boolField", DataTypes.BOOLEAN); + fields[6] = new Field("dateField", DataTypes.DATE); + fields[7] = new Field("timeField", DataTypes.TIMESTAMP); + fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2)); + fields[9] = new Field("varcharField", DataTypes.VARCHAR); + fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING)); + Map<String, String> map = new HashMap<>(); + map.put("complex_delimiter_level_1", "#"); + CarbonWriter writer = CarbonWriter.builder() + .outputPath(path) + .withLoadOptions(map) + .withCsvInput(new Schema(fields)).build(); + + for (int i = 0; i < 10; i++) { + String[] row2 = new String[]{ + "robot" + (i % 10), + String.valueOf(i % 10000), + String.valueOf(i), + String.valueOf(Long.MAX_VALUE - i), + String.valueOf((double) i / 2), + String.valueOf(true), + "2019-03-02", + "2019-02-12 03:03:34", + "12.345", + "varchar", + "Hello#World#From#Carbon" + }; + writer.write(row2); + } + writer.close(); + + File[] dataFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (name == null) { + return false; + } + return name.endsWith("carbondata"); + } + }); + if (dataFiles == null || dataFiles.length < 1) { + throw new RuntimeException("Carbon index file not exists."); + } + Schema schema = CarbonSchemaReader + .readSchemaInDataFile(dataFiles[0].getAbsolutePath()) + .asOriginOrder(); + // Transform the schema + String[] strings = new String[schema.getFields().length]; + for (int i = 0; i < schema.getFields().length; i++) { + strings[i] = (schema.getFields())[i].getFieldName(); + } + + // Read data + CarbonReader reader = CarbonReader + .builder(path, "_temp") + .projection(strings) + .build(); + + int i = 0; + while (reader.hasNext()) { + Object[] row = (Object[]) reader.readNextRow(); + assert (row[0].equals("robot" + i)); + assert (row[2].equals(i)); + assert (row[6].equals(17957)); + Object[] arr = (Object[]) row[10]; + assert (arr[0].equals("Hello")); + assert (arr[3].equals("Carbon")); + i++; + } + reader.close(); + FileUtils.deleteDirectory(new File(path)); + } catch (Throwable e) { + e.printStackTrace(); + } + } }
