Repository: carbondata Updated Branches: refs/heads/master dde0873f7 -> 161347155
[CARBONDATA-2393]TaskNo is not working for SDK Issue:- Task No is not getting reflected in the Carbon Data file and in index file . Cause :- Task No is getting overwritten in CarbonTableOutputformate even CarbonModel have taskNo. Solution :- if CarbomModel has taskNo then no need to overwrite . This closes #2224 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/16134715 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/16134715 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/16134715 Branch: refs/heads/master Commit: 161347155d9c429cf7275a617ace6dbc4c0fcdaf Parents: dde0873 Author: BJangir <[email protected]> Authored: Tue Apr 24 20:12:54 2018 +0530 Committer: kumarvishal09 <[email protected]> Committed: Mon Apr 30 20:13:12 2018 +0530 ---------------------------------------------------------------------- .../hadoop/api/CarbonTableOutputFormat.java | 8 +-- .../sdk/file/CSVCarbonWriterTest.java | 52 +++++++++++++++++++- 2 files changed, 56 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/16134715/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java index 43e0221..36ba02d 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java @@ -233,9 +233,11 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter( TaskAttemptContext taskAttemptContext) throws IOException { final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration()); - loadModel.setTaskNo(taskAttemptContext.getConfiguration().get( - "carbon.outputformat.taskno", - String.valueOf(System.nanoTime()))); + //if loadModel having taskNo already(like in SDK) then no need to overwrite + if (null == loadModel.getTaskNo() || loadModel.getTaskNo().isEmpty()) { + loadModel.setTaskNo(taskAttemptContext.getConfiguration() + .get("carbon.outputformat.taskno", String.valueOf(System.nanoTime()))); + } loadModel.setDataWritePath( taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath")); final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext); http://git-wip-us.apache.org/repos/asf/carbondata/blob/16134715/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index 76778e5..c4dcee9 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -205,7 +205,6 @@ public class CSVCarbonWriterTest { FileUtils.deleteDirectory(new File(path)); } - @Test(expected = IOException.class) public void testWhenWriterthrowsError() throws IOException{ CarbonWriter carbonWriter = null; @@ -246,4 +245,55 @@ public class CSVCarbonWriterTest { carbonWriter.close(); } + + @Test + public void testTaskNo() throws IOException { + // TODO: write all data type and read by CarbonRecordReader to verify the content + String path = "./testWriteFiles"; + FileUtils.deleteDirectory(new File(path)); + + Field[] fields = new Field[2]; + fields[0] = new Field("stringField", DataTypes.STRING); + fields[1] = new Field("intField", DataTypes.INT); + + + try { + CarbonWriterBuilder builder = CarbonWriter.builder() + .withSchema(new Schema(fields)) + .isTransactionalTable(true).taskNo("5") + .outputPath(path); + + CarbonWriter writer = builder.buildWriterForCSVInput(); + + for (int i = 0; i < 2; i++) { + String[] row = new String[]{ + "robot" + (i % 10), + String.valueOf(i) + }; + writer.write(row); + } + writer.close(); + + File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null")); + Assert.assertTrue(segmentFolder.exists()); + + File[] dataFiles = segmentFolder.listFiles(new FileFilter() { + @Override public boolean accept(File pathname) { + return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT); + } + }); + Assert.assertNotNull(dataFiles); + Assert.assertTrue(dataFiles.length > 0); + String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(dataFiles[0].getName()); + long taskID = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo); + Assert.assertEquals("Task Id is not matched", taskID, 5); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + finally { + FileUtils.deleteDirectory(new File(path)); + } + } + }
