Repository: carbondata Updated Branches: refs/heads/master a7faef8a0 -> 74770aa38
[CARBONDATA-2555] SDK reader set default isTransactional as false SDK writer is having default value of isTransactional is false. But reader is not like this. So, Fixing this by deafult make SDK to use flat folder structure. This closes #2352 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/74770aa3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/74770aa3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/74770aa3 Branch: refs/heads/master Commit: 74770aa38173985ece5527847f8e2446966e4fd7 Parents: a7faef8 Author: ajantha-bhat <[email protected]> Authored: Tue May 29 12:03:55 2018 +0530 Committer: kunal642 <[email protected]> Committed: Wed May 30 10:26:19 2018 +0530 ---------------------------------------------------------------------- docs/sdk-guide.md | 2 +- .../examples/sdk/CarbonReaderExample.java | 1 - .../carbondata/examples/sdk/SDKS3Example.java | 26 +++----------------- .../sdk/file/CarbonReaderBuilder.java | 4 +-- .../carbondata/sdk/file/CarbonReaderTest.java | 20 +++++++++++---- 5 files changed, 21 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/docs/sdk-guide.md ---------------------------------------------------------------------- diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md index 3c575fe..1d225a9 100644 --- a/docs/sdk-guide.md +++ b/docs/sdk-guide.md @@ -492,7 +492,7 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/ * Configure the transactional status of table * If set to false, then reads the carbondata and carbonindex files from a flat folder structure. * If set to true, then reads the carbondata and carbonindex files from segment folder structure. - * Default value is true + * Default value is false * * @param isTransactionalTable whether is transactional table or not * @return CarbonReaderBuilder object http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java index 937bfa0..d7886c0 100644 --- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java +++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java @@ -44,7 +44,6 @@ public class CarbonReaderExample { fields[1] = new Field("age", DataTypes.INT); CarbonWriter writer = CarbonWriter.builder() - .isTransactionalTable(true) .outputPath(path) .persistSchemaFile(true) .buildWriterForCSVInput(new Schema(fields)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java index 7fab2cc..80c56fc 100644 --- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java +++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java @@ -29,8 +29,8 @@ public class SDKS3Example { public static void main(String[] args) throws Exception { LogService logger = LogServiceFactory.getLogService(SDKS3Example.class.getName()); if (args == null || args.length < 3) { - logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>" + - "<s3-endpoint> [table-path-on-s3] [persistSchema] [transactionalTable]"); + logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>" + + "<s3-endpoint> [table-path-on-s3] [rows]"); System.exit(0); } @@ -44,24 +44,6 @@ public class SDKS3Example { num = Integer.parseInt(args[4]); } - Boolean persistSchema = true; - if (args.length > 5) { - if (args[5].equalsIgnoreCase("true")) { - persistSchema = true; - } else { - persistSchema = false; - } - } - - Boolean transactionalTable = true; - if (args.length > 6) { - if (args[6].equalsIgnoreCase("true")) { - transactionalTable = true; - } else { - transactionalTable = false; - } - } - Field[] fields = new Field[2]; fields[0] = new Field("name", DataTypes.STRING); fields[1] = new Field("age", DataTypes.INT); @@ -69,9 +51,7 @@ public class SDKS3Example { .setAccessKey(args[0]) .setSecretKey(args[1]) .setEndPoint(args[2]) - .outputPath(path) - .persistSchemaFile(persistSchema) - .isTransactionalTable(transactionalTable); + .outputPath(path); CarbonWriter writer = builder.buildWriterForCSVInput(new Schema(fields)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java index 1e73e8c..e99ff0d 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java @@ -48,7 +48,7 @@ public class CarbonReaderBuilder { private String[] projectionColumns; private Expression filterExpression; private String tableName; - private boolean isTransactionalTable = true; + private boolean isTransactionalTable; /** * It will be true if use the projectAllColumns methodï¼ @@ -84,7 +84,7 @@ public class CarbonReaderBuilder { * Configure the transactional status of table * If set to false, then reads the carbondata and carbonindex files from a flat folder structure. * If set to true, then reads the carbondata and carbonindex files from segment folder structure. - * Default value is true + * Default value is false * * @param isTransactionalTable whether is transactional table or not * @return CarbonReaderBuilder object http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index ee095a1..deb6d06 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -61,7 +61,7 @@ public class CarbonReaderTest extends TestCase { TestUtil.writeFilesAndVerify(new Schema(fields), path, true); - CarbonReader reader = CarbonReader.builder(path, "_temp") + CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true) .projection(new String[]{"name", "age"}).build(); // expected output after sorting @@ -87,6 +87,7 @@ public class CarbonReaderTest extends TestCase { // Read again CarbonReader reader2 = CarbonReader .builder(path, "_temp") + .isTransactionalTable(true) .projection(new String[]{"name", "age"}) .build(); @@ -118,6 +119,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") .projection(new String[]{"name", "name", "age", "name"}) + .isTransactionalTable(true) .build(); // expected output after sorting @@ -159,11 +161,13 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") .projection(new String[]{"name", "age"}) + .isTransactionalTable(true) .build(); // Reader 2 CarbonReader reader2 = CarbonReader .builder(path, "_temp") .projection(new String[]{"name", "age"}) + .isTransactionalTable(true) .build(); while (reader.hasNext()) { @@ -191,7 +195,7 @@ public class CarbonReaderTest extends TestCase { TestUtil.writeFilesAndVerify(new Schema(fields), path, true); - CarbonReader reader = CarbonReader.builder(path, "_temp") + CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true) .projection(new String[]{"name", "age"}).build(); reader.close(); @@ -305,7 +309,7 @@ public class CarbonReaderTest extends TestCase { // Write to a Non Transactional Table TestUtil.writeFilesAndVerify(new Schema(fields), path, true, false); - CarbonReader reader = CarbonReader.builder(path, "_temp") + CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true) .projection(new String[]{"name", "age"}) .isTransactionalTable(false) .build(); @@ -422,8 +426,8 @@ public class CarbonReaderTest extends TestCase { Assert.assertNotNull(dataFiles); Assert.assertTrue(dataFiles.length > 0); - CarbonReader reader = CarbonReader - .builder(path, "_temp") + CarbonReader reader = CarbonReader.builder(path, "_temp") + .isTransactionalTable(true) .projection(new String[]{ "stringField" , "shortField" @@ -548,6 +552,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") + .isTransactionalTable(true) .projection(strings) .build(); @@ -662,6 +667,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") .projection(strings) + .isTransactionalTable(true) .build(); int i = 0; @@ -766,6 +772,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") + .isTransactionalTable(true) .projection(strings) .build(); @@ -808,6 +815,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") + .isTransactionalTable(true) .projectAllColumns() .build(); @@ -846,6 +854,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") + .isTransactionalTable(true) .build(); // expected output after sorting @@ -881,6 +890,7 @@ public class CarbonReaderTest extends TestCase { CarbonReader reader = CarbonReader .builder(path, "_temp") .projection(new String[]{}) + .isTransactionalTable(true) .build(); assert (false); } catch (RuntimeException e) {
