Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2433#discussion_r199384025
--- Diff:
store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java
---
@@ -284,4 +284,137 @@ public void testSchemaPersistence() throws
IOException {
FileUtils.deleteDirectory(new File(path));
}
+ @Test
+ public void testLocalDictionarywithTrue() throws Exception {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("surname", DataTypes.STRING);
+ fields[2] = new Field("age", DataTypes.INT);
+
+ CarbonWriterBuilder builder =
CarbonWriter.builder().isTransactionalTable(false).sortBy(new
String[]{"name"}).withBlockSize(12).isLocalDictionaryEnabled(true)
+
.uniqueIdentifier(System.currentTimeMillis()).taskNo(System.nanoTime()).outputPath(path);
+ CarbonWriter carbonWriter = builder.buildWriterForCSVInput(new
Schema(fields));
+ for (int i = 0; i < 100; i++) {
+ carbonWriter.write(new String[]{"robot" + (i % 10),"robot_surname" +
(i % 10), String.valueOf(i)});
+ }
+ carbonWriter.close();
+
+ File segmentFolder = new File(path);
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return
pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertTrue(dataFiles.length > 0);
+
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testLocalDictionarywithFalseOption() throws Exception {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("surname", DataTypes.STRING);
+ fields[2] = new Field("age", DataTypes.INT);
+
+ CarbonWriterBuilder builder =
CarbonWriter.builder().isTransactionalTable(false).sortBy(new
String[]{"name"}).withBlockSize(12).isLocalDictionaryEnabled(false)
+
.uniqueIdentifier(System.currentTimeMillis()).taskNo(System.nanoTime()).outputPath(path);
+ CarbonWriter carbonWriter = builder.buildWriterForCSVInput(new
Schema(fields));
+ for (int i = 0; i < 100; i++) {
+ carbonWriter.write(new String[]{"robot" + (i % 10),"robot_surname" +
(i % 10), String.valueOf(i)});
+ }
+ carbonWriter.close();
+
+ File segmentFolder = new File(path);
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return
pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertTrue(dataFiles.length > 0);
+
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testLocalDictionarywithThreshold() throws Exception {
--- End diff --
can we add some validation to check whether fall back happened or not.?
---
